#-------------------------------------------------------------------------------
package OpenILS::Application::Storage::FTS;
use OpenSRF::Utils::Logger qw/:level/;
+ use Unicode::Normalize;
my $log = 'OpenSRF::Utils::Logger';
sub compile {
my $self = shift;
- my $term = shift;
+ my $term = NFD(shift());
$log->debug("Raw term: $term",DEBUG);
$self = ref($self) || $self;
$self = bless {} => $self;
- $term =~ s/\pM//gos;
+ $term =~ s/(\pM+)//gos;
$self->decompose($term);
my $newterm = join('&', $self->words);
use base qw/OpenILS::Application::Storage/;
use strict; use warnings;
+use Unicode::Normalize;
use OpenSRF::EX qw/:try/;
use OpenSRF::Utils::SettingsClient;
],
fixup => '
do {
- $text = lc($text);
+ $text = lc(NFD($text));
+ $text =~ s/\pM+//gso;
$text =~ s/\s+/ /sgo;
$text =~ s/^\s*(.+)\s*$/$1/sgo;
$text =~ s/\b(?:the|an?)\b//sgo;
$text =~ s/\[.[^\]]+\]//sgo;
$text =~ s/\s*[;\/\.]*$//sgo;
- $text =~ s/\pM+//gso;
};
',
},
],
fixup => '
do {
- $text = lc($text);
+ $text = lc(NFD($text));
+ $text =~ s/\pM+//gso;
$text =~ s/\s+/ /sgo;
$text =~ s/^\s*(.+)\s*$/$1/sgo;
$text =~ s/,?\s+.*$//sgo;
],
fixup => '
do {
- $text = lc($text);
+ $text = lc(NFD($text));
+ $text =~ s/\pM+//gso;
$text =~ s/\s+/ /sgo;
$text =~ s/^\s*(.+)\s*$/$1/sgo;
$text =~ s/\b(?:the|an?)\b//sgo;
],
fixup => '
do {
- $text = lc($text);
+ $text = lc(NFD($text));
+ $text =~ s/\pM+//gso;
$text =~ s/\s+/ /sgo;
$text =~ s/^\s*(.+)\s*$/$1/sgo;
$text =~ s/,?\s+.*$//sgo;
my $ns = new Fieldmapper::metabib::full_rec;
$ns->tag( 'LDR' );
- my $val = $tagline->textContent;
- $val =~ s/(\pM)//gso;
+ my $val = NFD($tagline->textContent);
+ $val =~ s/(\pM+)//gso;
$ns->value( $val );
push @ns_list, $ns;
my $ns = new Fieldmapper::metabib::full_rec;
$ns->tag( $tagline->getAttribute( "tag" ) );
- my $val = $tagline->textContent;
- $val =~ s/(\pM)//gso;
+ my $val = NFD($tagline->textContent);
+ $val =~ s/(\pM+)//gso;
$ns->value( $val );
push @ns_list, $ns;
$ns->ind1( $ind1 );
$ns->ind2( $ind2 );
$ns->subfield( $data->getAttribute( "code" ) );
- my $val = $data->textContent;
- $val =~ s/(\pM)//gso;
+ my $val = NFD($data->textContent);
+ $val =~ s/(\pM+)//gso;
$ns->value( lc($val) );
push @ns_list, $ns;
$string .= $value->textContent . " ";
}
}
+ $string = NFD($string);
$string =~ s/(\pM)//gso;
return lc($string);
}