5 use lib '/openils/lib/perl5/';
8 use OpenSRF::Application;
9 use OpenSRF::EX qw/:try/;
10 use OpenSRF::AppSession;
11 use OpenSRF::MultiSession;
12 use OpenSRF::Utils::SettingsClient;
13 use OpenILS::Application::AppUtils;
14 use OpenILS::Utils::Fieldmapper;
15 use Digest::MD5 qw/md5_hex/;
18 use Unicode::Normalize;
20 use Time::HiRes qw/time/;
25 use UNIVERSAL::require;
27 MARC::Charset->ignore_errors(1);
29 my ($utf8, $id_field, $count, $user, $password, $config, $keyfile, @files, @trash_fields) =
30 (0, '998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
33 'startid=i' => \$count,
34 'idfield=s' => \$id_field,
36 'password=s' => \$password,
37 'keyfile=s' => \$keyfile,
38 'config=s' => \$config,
40 'trash=s' => \@trash_fields,
43 @files = @ARGV if (!@files);
58 OpenSRF::System->bootstrap_client( config_file => $config );
59 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
61 $user = OpenILS::Application::AppUtils->check_user_session( login($user,$password) )->id;
65 open F, $keyfile or die "Couldn't open key file $keyfile";
67 if ( /^(\d+)\|(\S+)/o ) {
73 select STDERR; $| = 1;
74 select STDOUT; $| = 1;
76 my $batch = new MARC::Batch ( 'USMARC', @files );
78 $batch->warnings_off();
84 while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
89 my $field = $rec->field($id_field);
91 if ($field->is_control_field) {
94 $id = $field->subfield('a');
99 if ($id && $id =~ /(\d+)/o) {
106 if (my $tcn = $keymap{$id}) {
107 $rec->delete_field( $_ ) for ($rec->field($id_field));
108 $rec->append_fields( MARC::Field->new( $id_field, '', '', 'a', $tcn ) );
115 $rec = preprocess($rec);
116 $rec->delete_field( $_ ) for ($rec->field($id_field));
122 my $tcn_value = $rec->subfield('901' => 'a');
123 my $tcn_source = $rec->subfield('901' => 'b');
125 next if ($seen{$tcn_value});
126 $seen{$tcn_value} = 1;
128 (my $xml = $rec->as_xml_record()) =~ s/\n//sog;
129 $xml =~ s/^<\?xml.+\?\s*>//go;
130 $xml =~ s/>\s+</></go;
131 $xml =~ s/\p{Cc}//go;
132 $xml = entityize($xml);
134 my $bib = new Fieldmapper::biblio::record_entry;
139 $bib->creator($user);
140 $bib->create_date('now');
142 $bib->edit_date('now');
143 $bib->tcn_source($tcn_source);
144 $bib->tcn_value($tcn_value);
145 $bib->last_xact_id('IMPORT-'.$starttime);
147 print JSON->perl2JSON($bib)."\n";
151 if (!($count % 20)) {
152 print STDERR "\r$count\t". $count / (time - $starttime);
159 my ($id, $source, $value);
162 my $f = $rec->field($id_field);
163 $id = $f->subfield('a') if ($f);
167 my $f = $rec->field('001');
168 $id = $f->data if ($f);
172 my $f = $rec->field('000');
173 $id = $f->data if ($f);
174 $source = 'g'; # only PG seems to use this
178 my $f = $rec->field('020');
179 $id = $f->subfield('a') if ($f);
184 my $f = $rec->field('022');
185 $id = $f->subfield('a') if ($f);
190 my $f = $rec->field('010');
191 $id = $f->subfield('a') if ($f);
197 warn "\n !!! Record with no TCN : $count\n".$rec->as_formatted;
201 $rec->delete_field($_) for ($rec->field($id_field, @trash_fields));
205 $id =~ s/^(\S+).*$/$1/o;
207 $id = $source.$id if ($source);
209 ($source, $value) = $id =~ /^(.)(.+)$/o;
210 if ($id =~ /^o(\d+)$/o) {
215 my $tcn = MARC::Field->new(
218 b => do { $source_map{$source} || 'System' },
221 $rec->append_fields($tcn);
227 my( $username, $password, $type ) = @_;
231 my $seed = OpenILS::Application::AppUtils->simplereq(
233 'open-ils.auth.authenticate.init',
237 die("No auth seed. Couldn't talk to the auth server") unless $seed;
239 my $response = OpenILS::Application::AppUtils->simplereq(
241 'open-ils.auth.authenticate.complete',
242 { username => $username,
243 password => md5_hex($seed . md5_hex($password)),
246 die("No auth response returned on login.") unless $response;
248 my $authtime = $response->{payload}->{authtime};
249 my $authtoken = $response->{payload}->{authtoken};
251 die("Login failed for user $username!") unless $authtoken;
260 if ($form and $form eq 'D') {
261 $stuff = NFD($stuff);
263 $stuff = NFC($stuff);
266 $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;