###########################################################################
#
# lucenebuildproc.pm -- perl wrapper for building index with Lucene
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

package lucenebuildproc;

# This document processor outputs a document
# for lucene to process

# Use same basic XML structure setup by mgppbuilder/mgppbuildproc

use mgppbuildproc;
use ghtml;
use strict;
no strict 'refs'; # allow filehandles to be variables and viceversa


use IncrementalBuildUtils;

sub BEGIN {
    @lucenebuildproc::ISA = ('mgppbuildproc');
}


sub new {
    my $class = shift @_;
    my $self = new mgppbuildproc (@_);

    $self->{'numincdocs'} = 0;

    return bless $self, $class;
}


sub is_incremental_capable
{
    my $self = shift (@_);

    # Unlike MG and MGPP, Lucene supports incremental building
    return 1;
}


sub text {
    my $self = shift (@_);
    my ($doc_obj,$file) = @_;
    my $handle = $self->{'output_handle'};
    my $outhandle = $self->{'outhandle'};

    # only output this document if it is one to be indexed
    return if ($doc_obj->get_doc_type() ne "indexed_doc");

    my $indexed_doc = $self->is_subcollection_doc($doc_obj);

    # this is another document
    $self->{'num_docs'} += 1;

    # get the parameters for the output
    # split on : just in case there is subcoll and lang stuff
    my ($fields) = split (/:/, $self->{'index'});

    my $doc_level = $mgppbuildproc::level_map{'document'};
    my $gs2ns = 'xmlns:gs2="http://www.greenstone.org/gs2"';

    my $levels = $self->{'levels'};
    my $ldoc_level = $levels->{'document'};
    my $lsec_level = $levels->{'section'};
    my $lpar_level = $levels->{'paragraph'};

    my $docid="";
    if ($ldoc_level)
      {
	if ($self->{'gdbm_level'} eq 'document')
          {
	    my $doc_sec_num = $self->{'num_docs'};
	    $docid = "gs2:id=\"$doc_sec_num\"";
          }
        else
          {
	    # default is section level
	    my $doc_sec_num = $self->{'num_sections'} + 1;
	    $docid = "gs2:id=\"$doc_sec_num\"";
          }
      }
    my $documenttag = "<$doc_level $gs2ns file=\"$file\" $docid >\n";
    my $documentendtag = "\n</$doc_level>\n";

    my ($sectiontag) = "";
    if ($lsec_level)
      {
	$sectiontag = $mgppbuildproc::level_map{'section'};
      }
    my ($parastarttag) = "";
    my ($paraendtag) = "";
    if ($self->{'levels'}->{'paragraph'})
      {
	if ($self->{'strip_html'})
          {
	    $parastarttag = "<".$mgppbuildproc::level_map{'paragraph'}.">";
	    $paraendtag = "</".$mgppbuildproc::level_map{'paragraph'}.">";
          }
        else
          {
	    print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
          }
      }

    my $doc_section = 0; # just for this document

    my $text = "";
    $text .= $documenttag;
    # get the text for this document
    my $section = $doc_obj->get_top_section();
    while (defined $section)
      {
	# update a few statistics
	$doc_section++;
	$self->{'num_sections'}++;

	if ($sectiontag ne "")
          {
	    my $secid = "gs2:id=\"".$self->{'num_sections'}."\"";
	    $text .= "\n<$sectiontag $secid >\n";
          }

	# if we are doing subcollections, then some docs shouldn't be indexed.
	# but we need to put the section tag placeholders in there so the
	# sections match up with gdbm db
	my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
	if (($indexed_doc == 0) || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
	    $text .= "\n</$sectiontag>\n" if ($sectiontag ne "");
            $section = $doc_obj->get_next_section($section);
	    next;
          }

	$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
	foreach my $field (split (/;/, $fields))
          {
	    # only deal with this field if it doesn't start with top or
	    # this is the first section
	    my $real_field = $field;
	    next if (($real_field =~ s/^top//) && ($doc_section != 1));

	    my $new_text = "";
	    my $tmp_text = "";

	    # If allfields is requested add all metadata fields and text as
            # belonging to the ZZ field
	    if ($real_field eq "allfields") {
              # Text first - no html nor paragraph tags
              $new_text .= "$parastarttag<ZZ index=\"1\">\n";
              $tmp_text = $self->preprocess_text($doc_obj->get_text ($section), 1, "");
              &ghtml::htmlsafe($tmp_text);
              $new_text .= "$tmp_text</ZZ>$paraendtag\n";
              # Then Metadata
              my $metadata = $doc_obj->get_all_metadata ($section);
              foreach my $pair (@$metadata) {
                my ($mfield, $mvalue) = (@$pair);
                &ghtml::htmlsafe($mvalue);
                # check fields here, maybe others dont want - change to use dontindex!!
                if ($mfield ne "Identifier"
                    && $mfield !~ /^gsdl/
                    && $mfield ne "classifytype"
                    && $mfield ne "assocfilepath"
                    && defined $mvalue && $mvalue ne "") {
                  $new_text .= "$parastarttag<ZZ index=\"1\">$mvalue</ZZ>$paraendtag\n";
                }
                if (!defined $self->{'indexfields'}->{$mfield}) {
                  $self->{'indexfields'}->{$mfield} = 1;
                }
              }
	    }
	    # metadata - output all metadata we know about except gsdl stuff
	    elsif ($real_field eq "metadata" || $real_field eq "allfields") {
		my $shortname = "";
		my $metadata = $doc_obj->get_all_metadata ($section);
		foreach my $pair (@$metadata) {
		    my ($mfield, $mvalue) = (@$pair);
                    &ghtml::htmlsafe($mvalue);
		    # check fields here, maybe others dont want - change to use dontindex!!
		    if ($mfield ne "Identifier" 
			&& $mfield !~ /^gsdl/ 
			&& $mfield ne "classifytype"
			&& $mfield ne "assocfilepath"
			&& defined $mvalue && $mvalue ne "") {

			if (defined $self->{'indexfieldmap'}->{$mfield}) {
			    $shortname = $self->{'indexfieldmap'}->{$mfield};
			}
			else {
			    $shortname = $self->create_shortname($mfield);
			    $self->{'indexfieldmap'}->{$mfield} = $shortname;
			    $self->{'indexfieldmap'}->{$shortname} = 1;
			}
			$new_text .= "$parastarttag<$shortname index=\"1\">$mvalue</$shortname>$paraendtag\n";
			if (!defined $self->{'indexfields'}->{$mfield}) {
			    $self->{'indexfields'}->{$mfield} = 1;
			}
		    }
		}
	    }
	    else {
		#individual metadata and or text specified - could be a comma separated list
		my $shortname="";
		if (defined $self->{'indexfieldmap'}->{$real_field}) {
		    $shortname = $self->{'indexfieldmap'}->{$real_field};
		}
		else {
		    $shortname = $self->create_shortname($real_field);
		    $self->{'indexfieldmap'}->{$real_field} = $shortname;
		    $self->{'indexfieldmap'}->{$shortname} = 1;
		}

		my @metadata_list = ();
		foreach my $submeta (split /,/, $real_field) {
		    if ($submeta eq "text") {
			my $section_text = $doc_obj->get_text($section);
			if ($self->{'indexing_text'}) {
                            # tag the text with <Text>...</Text>, add the <Paragraph> tags and always strip out HTML
			    $new_text .= "$parastarttag<$shortname index=\"1\">\n";
			    if ($parastarttag ne "") {
				$section_text = $self->preprocess_text($section_text, 1, "</$shortname>$paraendtag$parastarttag<$shortname index=\"1\">");
			    }
			    else {
				# we don't want to individually tag each paragraph if not doing para indexing
				$section_text = $self->preprocess_text($section_text, 1, "");
			    }
			    $new_text .= "$section_text</$shortname>$paraendtag\n";
			}
			else { # leave html stuff in, but escape the tags, and dont add Paragraph tags - never retrieve paras at the moment
			    $tmp_text .= $doc_obj->get_text ($section);
			    &ghtml::htmlsafe($tmp_text);
			    $new_text .= $tmp_text;
			}
		    }
		    else {
			my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
			if ($section ne $doc_obj->get_top_section() && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
			    if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
				push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
			    }
			}
			push (@metadata_list, @section_metadata);
		    }
		}
		foreach my $item (@metadata_list) {
		    $new_text .= "$parastarttag<$shortname index=\"1\">$item</$shortname>$paraendtag\n";
		}
	    }
	    # filter the text
	    $self->filter_text ($field, $new_text);
	    $self->{'num_processed_bytes'} += length ($new_text);

	    $text .= "$new_text";
	} # foreach field

	$text .= "\n</$sectiontag>\n" if ($sectiontag ne "");

        $section = $doc_obj->get_next_section($section);
    } #while defined section
    print $handle "$text\n$documentendtag";
    #print STDOUT "$text\n$documentendtag";
}

# /** We make this builder pretend to be a document processor so we can get
#  *  information back from the plugins.
#  *
#  *  @param  $self    A reference to this Lucene builder
#  *  @param  $doc_obj A reference to a document object representing what was
#  *                   parsed by the GAPlug
#  *  @param  $file    The name of the file parsed as a string
#  *
#  *  @author John Thompson, DL Consulting Ltd
#  */
sub process()
  {
    my $self = shift (@_);
    my ($doc_obj, $file) = @_;

    # If this is called from any stage other than an incremental infodb we want
    # to pass through to the superclass of build
    if ($self->get_mode() eq "incinfodb")
      {
        print STDERR "*** Processing a document added using INCINFODB ***\n";
        my ($archivedir) = $file =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
        $archivedir = "" unless defined $archivedir;
        $archivedir =~ s/\\/\//g;
        $archivedir =~ s/^\/+//;
        $archivedir =~ s/\/+$//;

        # Number of files
        print STDERR "There are " . scalar($doc_obj->get_assoc_files()) . " associated documents...\n";

        # resolve the final filenames of the files associated with this document
        $self->assoc_files ($doc_obj, $archivedir);

        # is this a paged or a hierarchical document
        my ($thistype, $childtype) = $self->get_document_type ($doc_obj);

        # Determine the actual docnum by checking if we've processed any
        # previous incrementally added documents. If so, carry on from there.
        # Otherwise we set the counter to be the same as the number of
        # sections encountered during the previous build
        if ($self->{'numincdocs'} == 0)
          {
            $self->{'numincdocs'} = $self->{'starting_num_sections'} + 1;
          }

        my $section = $doc_obj->get_top_section ();
        print STDERR "+ top section: '$section'\n";
        my $doc_OID = $doc_obj->get_OID();
        my $url = "";
        while (defined $section)
          {
            print STDERR "+ processing section: '$section'\n";
            # Attach all the other metadata to this document
            # output the fact that this document is a document (unless doctype
            # has been set to something else from within a plugin
            my $dtype = $doc_obj->get_metadata_element ($section, "doctype");
            if (!defined $dtype || $dtype !~ /\w/)
              {
                $doc_obj->add_utf8_metadata($section, "doctype", $dtype);
              }
            # output whether this node contains text
            if ($doc_obj->get_text_length($section) > 0)
              {
                $doc_obj->add_utf8_metadata($section, "hastxt", 1);
              }
            else
              {
                $doc_obj->add_utf8_metadata($section, "hastxt", 0);
              }

            # output archivedir if at top level
            if ($section eq $doc_obj->get_top_section())
              {
                $doc_obj->add_utf8_metadata($section, "archivedir", $archivedir);
		$doc_obj->add_utf8_metadata($section, "thistype", $thistype);
              }

            # output a list of children
            my $children = $doc_obj->get_children ($section);
            if (scalar(@$children) > 0)
              {
                $doc_obj->add_utf8_metadata($section, "childtype", $childtype);
                my @contains = ();
                foreach my $child (@$children)
                  {
                    if ($child =~ /^.*?\.(\d+)$/)
                      {
                        push (@contains, "\".$1");
                      }
                    else
                      {
                        push (@contains, "\".$child");
                      }
                  }
                $doc_obj->add_utf8_metadata($section, "contains", join(";", @contains));
              }
            #output the matching doc number
            print STDERR "+ docnum=" . $self->{'numincdocs'} . "\n";
            $doc_obj->add_utf8_metadata($section, "docnum", $self->{'numincdocs'});

            $self->{'numincdocs'}++;
            $section = $doc_obj->get_next_section($section);
            # if no sections wanted, only gdbm the docs
            last if ($self->{'gdbm_level'} eq "document");
          }
        print STDERR "\n*** incrementally add metadata from document at: " . $file . "\n";
        &IncrementalBuildUtils::addDocument($self->{'collection'}, $doc_obj, $doc_obj->get_top_section());
      }
    else
      {
        $self->mgppbuildproc::process(@_);
      }
  }
# /** process() **/

sub get_num_docs {
    my $self = shift (@_);
    #rint STDERR "get_num_docs(): $self->{'num_docs'}\n";
    return $self->{'num_docs'};
}

sub get_num_sections {
    my $self = shift (@_);
    #rint STDERR "get_num_sections(): $self->{'num_sections'}\n";
    return $self->{'num_sections'};
}

# num_bytes is the actual number of bytes in the collection
# this is normally the same as what's processed during text compression
sub get_num_bytes {
    my $self = shift (@_);
    #rint STDERR "get_num_bytes(): $self->{'num_bytes'}\n";
    return $self->{'num_bytes'};
}

1;