Lucene Facet Search in .NET

["Udayakumar, Jay" <Ud...@DNB.com.INVALID>]

Hi Apache Team,

How are you? Please find the below code for your reference:

Build Lucene Index

Private Sub BuildIndex()

        Try
            'use the version 30, but not 29.
            Dim version As Lucene.Net.Util.Version = Lucene.Net.Util.Version.LUCENE_30
            line = 1

            Dim cloudStorageAccount As CloudStorageAccount = cloudStorageAccount.DevelopmentStorageAccount
            line = 2

            'connection string from app.config with primary key
            cloudStorageAccount.TryParse(My.Settings.blobStorage, cloudStorageAccount)
            line = 3

            'don't use special characters in the below string
            Dim azureDirectory As New RAMDirectory()

            line = 4

            Dim findexExists As Boolean = IndexReader.IndexExists(azureDirectory)
            line = 5

            Dim indexWriter As New IndexWriter(azureDirectory, New StandardAnalyzer(Util.Version.LUCENE_30), True, _
                                               New Lucene.Net.Index.IndexWriter.MaxFieldLength(indexWriter.DEFAULT_MAX_FIELD_LENGTH))
            line = 6

            'Create a DataSet and fill it from SQL Database
            Dim ds As New DataSet()

            Using sqlCon As New SqlConnection(Cs_DUNSLink)
                sqlCon.Open()

                Dim sqlCmd As New SqlCommand()
                sqlCmd.Connection = sqlCon

                sqlCmd.CommandType = CommandType.Text

                'Only get the minimum fields we need; Bio to index, Id so search results
                'can look up the record in SQL Database
                sqlCmd.CommandText = "SELECT FEI_Num, dnb_duns_nbr FROM FMLS_LOOKUP"  'add the fields we need in ui

                Dim sqlAdap As New SqlDataAdapter(sqlCmd)
                sqlAdap.Fill(ds)
            End Using

            line = 7
            If ds.Tables(0) IsNot Nothing Then
                Dim dt As DataTable = ds.Tables(0)

                If dt.Rows.Count > 0 Then
                    For Each dr As DataRow In dt.Rows
                        Call AddDoc(indexWriter, dr.Item("FEI_Num").ToString, dr.Item("dnb_duns_nbr").ToString)
                    Next
                End If
            End If

            line = 8

            ' Close the writer
            indexWriter.Close()
            line = 9

            Call SendEmail(ApplicationEmail, ApplicationName & " - Build Lucene Index (Success)", "Lucene Index has been built successfully for FMLS Lookup Table")

        Catch ex As Exception
            Throw ex

        Finally

        End Try

End Sub


Search Lucene in Portal

  Private Sub SearchLucene(ByVal SearchField As String, ByVal SearchString As String)

        Try
            'Dim parser As QueryParser = New QueryParser("Title", New StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30))
            'parser.Query("FEI_Num")

            If SearchField = "All" And SearchString = "All" Then
                _query = New MatchAllDocsQuery()
            Else
                _query = New QueryParser(Lucene.Net.Util.Version.LUCENE_30, "dnb_duns_nbr", New StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)).Parse("116005893")
            End If

            Response.Write("Query: " & _query.ToString)

            ' pass in the reader and the names of the facets that you've created using fields in the documents.
            ' the facets are determined
            sfs = New SimpleFacetedSearch(_reader, "dnb_duns_nbr")

            Response.Write("; SimpleFacetedSearch: " & sfs.ToString)

            ' then pass in the query into the search like you normally would with a typical search class.
            hits = sfs.Search(_query)
            Response.Write("; Hits: " & hits.TotalHitCount)

            ' what comes back is different than normal.
            ' the result documents & hits are grouped by facets.
            ' you'll need to iterate over groups of hits-per-facet.
            totalHits = hits.TotalHitCount

            Response.Write("; Hits.HitsPerFacet.Count: " & hits.HitsPerFacet.Count)

            For Each hpg In hits.HitsPerFacet
                Dim hitCountPerGroup As Long = hpg.HitCount
                Dim facetName As SimpleFacetedSearch.FacetName = hpg.Name

                For Each doc As Document In hpg.Documents
                    Dim text As String = doc.GetField("text").StringValue()
                    Response.Write("; Text: " & text)
                    'System.Diagnostics.Debug.WriteLine(Convert.ToString(">>" & facetName.ToString & ": ") & text)
                Next
            Next

            Response.End()

            'Dim q As TermQuery = New TermQuery(New Term("FEI_Num", "1"))
            'Dim t As Term = q.Term

            'Dim str As String = t.Field(0)
            'Dim text As String = t.Text(0)

            'Dim query As Lucene.Net.Search.Query = parser.Parse("FEI_Number:(1 AND 2)")

            'Dim hits As SimpleFacetedSearch.Hits = searcher.Search(query, )

            ''Dim ObjDocs As TopDocs = searcher.Search(query, 100)


            'For i As Integer = 0 To ObjDocs.TotalHits - 1
            '    Dim doc As Document = ObjDocs.Sco

            '    Console.WriteLine(doc.GetField("Title").StringValue())
            'Next

            ' Create the QueryParser, setting the default search field to 'Bio'
            'Dim parser As QueryParser(New StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30))

            ' Create a query from the Parser
            'Dim query As Query = parser.Parse(searchString)

            ' Retrieve matching hits
            'Dim hits As Hits = indexSearcher.Search(query)

            ' Loop through the matching hits, retrieving the document
            'For i As Integer = 0 To hits.Length() - 1
            '    'Retrieve the string value of the 'Id' field from the
            '    'hits.Doc(i) document.
            '    TextBox_Results.Text += "Id: " + hits.Doc(i).GetField("Id").StringValue() + vbLf
            'Next

        Catch ex As Exception
            ErrMsg = ex.ToString
            ErrMsg &= "; Query: " & _query.ToString
            ErrMsg &= "; SimpleFacetSearch: " & sfs.ToString
            ErrMsg &= "; Hits.TotalHitCount: " & hits.TotalHitCount
            ErrMsg &= "; hits.HitsPerFacet: " & hits.HitsPerFacet.ToString

            Call InsertErrorLog(ErrMsg, LoggedIn_EmailID)

            'Throw ex

        End Try

    End Sub

When I run F5 (in BuildDnBList.aspx) - laptop, it's throwing error in "sfs = New SimpleFacetedSearch(_reader, "dnb_duns_nbr")" as "OutOfMemory"

In Azure WebApp, I increased the capacity from 1.5GB (Default) to 7GB. Then, it wrote only "Query: *:*" (line 244), not any of the Response.Write statements (beyond line 248), means some error in the same line, but weird is no exceptions thrown.

Can you please help me to proceed further? I am available to take call at 571-353-4669 and Lync at UdayakumarJ@dnb.com<ma...@dnb.com>.

Thanks,
Jay