5 use lib '/openils/lib/perl5/';
8 use OpenSRF::Application;
9 use OpenSRF::EX qw/:try/;
10 use OpenSRF::AppSession;
11 use OpenSRF::MultiSession;
12 use OpenSRF::Utils::SettingsClient;
13 use OpenILS::Application::AppUtils;
14 use OpenILS::Utils::Fieldmapper;
15 use Digest::MD5 qw/md5_hex/;
18 use Unicode::Normalize;
20 use Time::HiRes qw/time/;
25 use UNIVERSAL::require;
27 MARC::Charset->ignore_errors(1);
29 my ($id_field, $count, $user, $password, $config, $keyfile, @files, @trash_fields) =
30 ('998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
33 'startid=i' => \$count,
34 'idfield=s' => \$id_field,
36 'password=s' => \$password,
37 'keyfile=s' => \$keyfile,
38 'config=s' => \$config,
40 'trash=s' => \@trash_fields,
43 @files = @ARGV if (!@files);
58 OpenSRF::System->bootstrap_client( config_file => $config );
59 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
61 $user = OpenILS::Application::AppUtils->check_user_session( login($user,$password) )->id;
65 open F, $keyfile or die "Couldn't open key file $keyfile";
67 if ( /^(\d+)\|(\S+)/o ) {
73 select STDERR; $| = 1;
74 select STDOUT; $| = 1;
76 my $batch = new MARC::Batch ( 'USMARC', @files );
78 $batch->warnings_off();
81 while ( my $rec = $batch->next ) {
84 my $field = $rec->field($id_field);
87 if ($field->is_control_field) {
90 $id = $field->subfield('a');
96 if ($id =~ /(\d+)/o) {
101 if (my $tcn = $keymap{$id}) {
102 $rec->delete_field( $_ ) for ($rec->field($id_field));
103 $rec->append_fields( MARC::Field->new( $id_field, '', '', 'a', $tcn ) );
110 $rec = preprocess($rec);
116 my $tcn_value = $rec->subfield($id_field => 'a');
117 my $tcn_source = $rec->subfield($id_field => 'b');
119 (my $xml = $rec->as_xml_record()) =~ s/\n//sog;
120 $xml =~ s/^<\?xml.+\?\s*>//go;
121 $xml =~ s/>\s+</></go;
122 $xml =~ s/\p{Cc}//go;
123 $xml = entityize($xml);
125 my $bib = new Fieldmapper::biblio::record_entry;
130 $bib->creator($user);
131 $bib->create_date('now');
133 $bib->edit_date('now');
134 $bib->tcn_source($tcn_source);
135 $bib->tcn_value($tcn_value);
136 $bib->last_xact_id('IMPORT-'.$starttime);
138 print JSON->perl2JSON($bib)."\n";
142 if (!($count % 20)) {
143 print STDERR "\r$count\t". $count / (time - $starttime);
150 my ($id, $source, $value);
153 my $f = $rec->field('001');
154 $id = $f->data if ($f);
158 my $f = $rec->field('000');
159 $id = 'g'.$f->data if ($f);
164 my $f = $rec->field('020');
165 $id = $f->subfield('a') if ($f);
170 my $f = $rec->field('022');
171 $id = $f->subfield('a') if ($f);
176 my $f = $rec->field('010');
177 $id = $f->subfield('a') if ($f);
182 my $f = $rec->field($id_field);
183 $id = $f->subfield('a') if ($f);
189 warn "\n !!! Record with no TCN : $count\n".$rec->as_formatted;
193 $rec->delete_field($_) for ($rec->field($id_field, @trash_fields));
197 $id =~ s/(\S+)$/$1/o;
199 $id = $source.$id if ($source);
201 ($source, $value) = $id =~ /^(.)(.+)$/o;
202 if ($id =~ /^o(\d+)$/o) {
207 my $tcn = MARC::Field->new(
211 'b', do { $source_map{$source} || 'System' },
214 $rec->append_fields($tcn);
220 my( $username, $password, $type ) = @_;
224 my $seed = OpenILS::Application::AppUtils->simplereq(
226 'open-ils.auth.authenticate.init',
230 die("No auth seed. Couldn't talk to the auth server") unless $seed;
232 my $response = OpenILS::Application::AppUtils->simplereq(
234 'open-ils.auth.authenticate.complete',
235 { username => $username,
236 password => md5_hex($seed . md5_hex($password)),
239 die("No auth response returned on login.") unless $response;
241 my $authtime = $response->{payload}->{authtime};
242 my $authtoken = $response->{payload}->{authtoken};
244 die("Login failed for user $username!") unless $authtoken;
253 if ($form and $form eq 'D') {
254 $stuff = NFD($stuff);
256 $stuff = NFC($stuff);
259 $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;