5 use lib '/openils/lib/perl5/';
8 use OpenSRF::Application;
9 use OpenSRF::EX qw/:try/;
10 use OpenSRF::AppSession;
11 use OpenSRF::MultiSession;
12 use OpenSRF::Utils::SettingsClient;
13 use OpenILS::Application::AppUtils;
14 use OpenILS::Utils::Fieldmapper;
15 use Digest::MD5 qw/md5_hex/;
18 use Unicode::Normalize;
20 use Time::HiRes qw/time/;
25 use UNIVERSAL::require;
27 MARC::Charset->ignore_errors(1);
29 my ($utf8, $id_field, $count, $user, $password, $config, $keyfile, @files, @trash_fields) =
30 (0, '998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
33 'startid=i' => \$count,
34 'idfield=s' => \$id_field,
36 'password=s' => \$password,
37 'keyfile=s' => \$keyfile,
38 'config=s' => \$config,
40 'trash=s' => \@trash_fields,
43 @files = @ARGV if (!@files);
58 OpenSRF::System->bootstrap_client( config_file => $config );
59 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
61 $user = OpenILS::Application::AppUtils->check_user_session( login($user,$password) )->id;
65 open F, $keyfile or die "Couldn't open key file $keyfile";
67 if ( /^(\d+)\|(\S+)/o ) {
73 select STDERR; $| = 1;
74 select STDOUT; $| = 1;
76 my $batch = new MARC::Batch ( 'USMARC', @files );
78 $batch->warnings_off();
82 while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
87 my $field = $rec->field($id_field);
89 if ($field->is_control_field) {
92 $id = $field->subfield('a');
97 if ($id && $id =~ /(\d+)/o) {
104 if (my $tcn = $keymap{$id}) {
105 $rec->delete_field( $_ ) for ($rec->field($id_field));
106 $rec->append_fields( MARC::Field->new( $id_field, '', '', 'a', $tcn ) );
113 $rec = preprocess($rec);
119 my $tcn_value = $rec->subfield($id_field => 'a');
120 my $tcn_source = $rec->subfield($id_field => 'b');
122 (my $xml = $rec->as_xml_record()) =~ s/\n//sog;
123 $xml =~ s/^<\?xml.+\?\s*>//go;
124 $xml =~ s/>\s+</></go;
125 $xml =~ s/\p{Cc}//go;
126 $xml = entityize($xml);
128 my $bib = new Fieldmapper::biblio::record_entry;
133 $bib->creator($user);
134 $bib->create_date('now');
136 $bib->edit_date('now');
137 $bib->tcn_source($tcn_source);
138 $bib->tcn_value($tcn_value);
139 $bib->last_xact_id('IMPORT-'.$starttime);
141 print JSON->perl2JSON($bib)."\n";
145 if (!($count % 20)) {
146 print STDERR "\r$count\t". $count / (time - $starttime);
153 my ($id, $source, $value);
156 my $f = $rec->field($id_field);
157 $id = $f->subfield('a') if ($f);
161 my $f = $rec->field('001');
162 $id = $f->data if ($f);
166 my $f = $rec->field('000');
167 $id = $f->data if ($f);
168 $source = 'g'; # only PG seems to use this
172 my $f = $rec->field('020');
173 $id = $f->subfield('a') if ($f);
178 my $f = $rec->field('022');
179 $id = $f->subfield('a') if ($f);
184 my $f = $rec->field('010');
185 $id = $f->subfield('a') if ($f);
191 warn "\n !!! Record with no TCN : $count\n".$rec->as_formatted;
195 $rec->delete_field($_) for ($rec->field($id_field, @trash_fields));
199 $id =~ s/^(\S+).*$/$1/o;
201 $id = $source.$id if ($source);
203 ($source, $value) = $id =~ /^(.)(.+)$/o;
204 if ($id =~ /^o(\d+)$/o) {
209 my $tcn = MARC::Field->new(
213 'b', do { $source_map{$source} || 'System' },
216 $rec->append_fields($tcn);
222 my( $username, $password, $type ) = @_;
226 my $seed = OpenILS::Application::AppUtils->simplereq(
228 'open-ils.auth.authenticate.init',
232 die("No auth seed. Couldn't talk to the auth server") unless $seed;
234 my $response = OpenILS::Application::AppUtils->simplereq(
236 'open-ils.auth.authenticate.complete',
237 { username => $username,
238 password => md5_hex($seed . md5_hex($password)),
241 die("No auth response returned on login.") unless $response;
243 my $authtime = $response->{payload}->{authtime};
244 my $authtoken = $response->{payload}->{authtoken};
246 die("Login failed for user $username!") unless $authtoken;
255 if ($form and $form eq 'D') {
256 $stuff = NFD($stuff);
258 $stuff = NFC($stuff);
261 $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;