From 7456f05a99c228c386cf8c2e0debad8ae09df669 Mon Sep 17 00:00:00 2001
From: miker <miker@dcc99617-32d9-48b4-a31d-7c20da2025e4>
Date: Mon, 17 Jul 2006 07:39:35 +0000
Subject: [PATCH] pipline fixups; integer qualities only, please

git-svn-id: svn://svn.open-ils.org/ILS/trunk@5030 dcc99617-32d9-48b4-a31d-7c20da2025e4
---
 Open-ILS/src/extras/import/direct_ingest.pl   |  1 +
 Open-ILS/src/extras/import/marc2bre.pl        | 48 ++++++++++---------
 Open-ILS/src/extras/import/pg_loader.pl       | 12 ++++-
 .../perlmods/OpenILS/Application/Ingest.pm    |  1 +
 4 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/Open-ILS/src/extras/import/direct_ingest.pl b/Open-ILS/src/extras/import/direct_ingest.pl
index 81fceca81b..dff72abdea 100755
--- a/Open-ILS/src/extras/import/direct_ingest.pl
+++ b/Open-ILS/src/extras/import/direct_ingest.pl
@@ -118,6 +118,7 @@ while ( my $rec = <> ) {
 	$count++;
 }
 
+$ses[$_]->close for (@ses);
 sub postprocess {
 	my $data = shift;
 	my $f = shift;
diff --git a/Open-ILS/src/extras/import/marc2bre.pl b/Open-ILS/src/extras/import/marc2bre.pl
index 53b23233cb..1e84ab1be3 100755
--- a/Open-ILS/src/extras/import/marc2bre.pl
+++ b/Open-ILS/src/extras/import/marc2bre.pl
@@ -26,8 +26,8 @@ use UNIVERSAL::require;
 
 MARC::Charset->ignore_errors(1);
 
-my ($id_field, $count, $user, $password, $config, $keyfile,  @files, @trash_fields) =
-	('998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
+my ($utf8, $id_field, $count, $user, $password, $config, $keyfile,  @files, @trash_fields) =
+	(0, '998', 1, 'admin', 'open-ils', '/openils/conf/bootstrap.conf');
 
 GetOptions(
 	'startid=i'	=> \$count,
@@ -49,7 +49,7 @@ my %processing_cache;
 my %source_map = (      
 	o  => 'OCLC',
 	i  => 'ISxN',    
-	l  => 'Local',
+	l  => 'LCCN',
 	s  => 'System',  
 	g  => 'Gutenberg',  
 );                              
@@ -78,23 +78,26 @@ $batch->strict_off();
 $batch->warnings_off();
 
 my $starttime = time;
-while ( my $rec = $batch->next ) {
-
+my $rec;
+while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
+	next if ($rec == -1);
 	my $id;
-	my $field = $rec->field($id_field);
 
-	if ($field) {
-		if ($field->is_control_field) {
-			$id = $field->data;
-		} else {
-			$id = $field->subfield('a');
+	if ($id_field) {
+		my $field = $rec->field($id_field);
+		if ($field) {
+			if ($field->is_control_field) {
+				$id = $field->data;
+			} else {
+				$id = $field->subfield('a');
+			}
 		}
-	} else {
-		$id = $count;
 	}
 		
-	if ($id =~ /(\d+)/o) {
+	if ($id && $id =~ /(\d+)/o) {
 		$id = $1;
+	} else {
+		$id = $count;
 	}
 
 	if ($keyfile) {
@@ -149,6 +152,11 @@ sub preprocess {
 
 	my ($id, $source, $value);
 
+        if (!$id) {
+                my $f = $rec->field($id_field);
+                $id = $f->subfield('a') if ($f);
+        }
+
 	if (!$id) {
 		my $f = $rec->field('001');
 		$id = $f->data if ($f);
@@ -156,8 +164,8 @@ sub preprocess {
 
 	if (!$id) {
 		my $f = $rec->field('000');
-		$id = 'g'.$f->data if ($f);
-		$source = 'g';
+		$id = $f->data if ($f);
+		$source = 'g'; # only PG seems to use this
 	}
 
         if (!$id) {
@@ -178,12 +186,6 @@ sub preprocess {
 		$source = 'l';
         }
 
-        if (!$id) {
-                my $f = $rec->field($id_field);
-                $id = $f->subfield('a') if ($f);
-		$source = 's';
-        }
-
 	if (!$id) {
 		$count++;
 		warn "\n !!! Record with no TCN : $count\n".$rec->as_formatted;
@@ -194,7 +196,7 @@ sub preprocess {
 
 	$id =~ s/\s*$//o;
 	$id =~ s/^\s*//o;
-	$id =~ s/(\S+)$/$1/o;
+	$id =~ s/^(\S+).*$/$1/o;
 
 	$id = $source.$id if ($source);
 
diff --git a/Open-ILS/src/extras/import/pg_loader.pl b/Open-ILS/src/extras/import/pg_loader.pl
index 86a9f1a2f1..a1f8552388 100755
--- a/Open-ILS/src/extras/import/pg_loader.pl
+++ b/Open-ILS/src/extras/import/pg_loader.pl
@@ -84,14 +84,22 @@ for my $h (@order) {
 
 	for my $line (@{ $lineset{$h} }) {
 		my @data;
+		my $x = 0;
 		for my $d (@$line) {
 			if (!defined($d)) {
 				$d = '\N';
 			} else {
-				$d =~ s/\t/\\t/go;
-				$d =~ s/\\/\\\\/go;
+				$d =~ s/\f/\\f/gos;
+				$d =~ s/\n/\\n/gos;
+				$d =~ s/\r/\\r/gos;
+				$d =~ s/\t/\\t/gos;
+				$d =~ s/\\/\\\\/gos;
+			}
+			if ($h eq 'bre' and $fieldcache{$h}{fields}[$x] eq 'quality') {
+				$d = int($d);
 			}
 			push @data, $d;
+			$x++;
 		}
 		$output->print( join("\t", @data)."\n" );
 	}
diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
index 4e19dfaee8..fe215b442b 100644
--- a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm
@@ -779,6 +779,7 @@ sub biblio_fingerprint_record {
 
 	my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
 	$log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
+	$fp->{quality} = int($fp->{quality});
 	return $fp;
 }
 __PACKAGE__->register_method(  
-- 
2.43.2