Lucene sort unexpected docvalues type NONE for field 'date'

Trying to sort the Lucene search results by a string field, which in my case is a string that represents the date.

And get this error

 
Exception in thread "main" java.lang.IllegalStateException: unexpected docvalues type NONE for field 'date' (expected=SORTED). Use UninvertingReader or index with docvalues.
    at org.apache.lucene.index.DocValues.checkField(DocValues.java:208)
    at org.apache.lucene.index.DocValues.getSorted(DocValues.java:264)
    at org.apache.lucene.search.FieldComparator$TermOrdValComparator.getSortedDocValues(FieldComparator.java:762)
    at org.apache.lucene.search.FieldComparator$TermOrdValComparator.getLeafComparator(FieldComparator.java:767)
    at org.apache.lucene.search.FieldValueHitQueue.getComparators(FieldValueHitQueue.java:183)
    at org.apache.lucene.search.TopFieldCollector$ScoringMaxScoreCollector.getLeafCollector(TopFieldCollector.java:328)
    at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:763)
 

The example code is shown below:

 
package com.makble.luceneexample;
 
import java.io.IOException;
 
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
 
public class SortExample {
    public static Analyzer analyzer = new StandardAnalyzer();
    public static IndexWriterConfig config = new IndexWriterConfig(analyzer);
    public static RAMDirectory ramDirectory = new RAMDirectory();
    public static IndexWriter indexWriter;
 
    public static void main(String args []) throws ParseException {
        createIndex();
        searchSingleTerm("title","lucene");
        ramDirectory.close();
    }
 
    public static void createDoc(String author, String title, String date) throws IOException {
        Document doc = new Document();
        doc.add(new TextField("author", author, Field.Store.YES));
        doc.add(new TextField("title", title, Field.Store.YES));
        doc.add(new Field ("date", date, Field.Store.YES, Field.Index.NOT_ANALYZED));
 
        indexWriter.addDocument(doc);
    }
 
    public static void createIndex() {
        try {
                indexWriter = new IndexWriter(ramDirectory, config);    
                createDoc("Sam", "Lucece index option analyzed vs not analyzed", "2016-12-12 20:19:57");    
                createDoc("Sam", "Luaene field boost and query time boost example", "2016-03-16 16:57:44");    
                createDoc("Jack", "How to do Lxeeci search highlight example", "2016-03-16 17:47:38");
                createDoc("Smith","Lucene BooleanQuery is depreacted as of 5.3.0" , "2015-04-30 11:44:25");
                createDoc("Smith","What is term vector in Lucene", "2015-04-10 20:33:53" );
 
                indexWriter.close();
        } catch (IOException | NullPointerException ex) {
            System.out.println("Exception : " + ex.getLocalizedMessage());
        } 
    }
 
    public static void searchIndexAndDisplayResults(Query query) {
        try {
            IndexReader idxReader = DirectoryReader.open(ramDirectory);
            IndexSearcher idxSearcher = new IndexSearcher(idxReader);
 
            Sort sort = new Sort(SortField.FIELD_SCORE,
                    new SortField("date", Type.STRING));
 
            TopDocs docs = idxSearcher.search(query, 10, sort,true, true);
            System.out.println("length of top docs: " + docs.scoreDocs.length);
            for (ScoreDoc doc : docs.scoreDocs) {
                Document thisDoc = idxSearcher.doc(doc.doc);
                System.out.println(doc.doc + "\t" + thisDoc.get("author")
                        + "\t" + thisDoc.get("title")
                        + "\t" + thisDoc.get("date"));
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
        }
    }
 
    public static void searchSingleTerm(String field, String termText){
        Term term = new Term(field, termText);
        TermQuery termQuery = new TermQuery(term);
 
        searchIndexAndDisplayResults(termQuery);
    }
 
 }
 
 

In Lucene 5, the sorting field use a very different kind of field. A simple not analyzed stored field is not enough. Instead you need to add the field with the type SortedDocValuesField.

Change the add method call solves the problem but it won't store the value of the date field.

 
doc.add(new SortedDocValuesField ("date", new BytesRef(date) ));
 

You will see the document.get("date") return null.

 
length of top docs: 4 sort by: <score>,<string: "date">
4    Smith    What is term vector in Lucene    null
3    Smith    Lucene BooleanQuery is depreacted as of 5.3.0    null
1    Sam    Lucene field boost and query time boost example    null
2    Jack    How to do Lucene search highlight example    null
 

According to SortedDocValuesField . You need add a StoredField with the same field name if you want to get the value from matched document.

 
doc.add(new SortedDocValuesField ("date", new BytesRef(date) ));
doc.add(new StoredField("date", date));
 

I get this, the SortedDocValuesField is dedicated for sorting, it actually won't store value in document, the class didn't even provide the parameter options. I think it may not a good abstraction from the viewpoint from the users.

It's new feature of Lucene 5. Sorting field using a new data structure. It's a column based.

Instead to look for value in document, now it's inverted again, you find document by lookup the values.