From 7266b044a6f47f9ee82d73a077dfc2615472e16f Mon Sep 17 00:00:00 2001
From: erickson <erickson@dcc99617-32d9-48b4-a31d-7c20da2025e4>
Date: Tue, 23 Sep 2008 15:32:28 +0000
Subject: [PATCH] moved to file-based marc storage, since memcache has a
 hard-coded 1MB limit on data sizes.  added a (hard-coded for now) max file
 size setting.  deleting file and cache data after records are spooled.  added
 example config

git-svn-id: svn://svn.open-ils.org/ILS/trunk@10689 dcc99617-32d9-48b4-a31d-7c20da2025e4
---
 Open-ILS/examples/opensrf.xml.example         |  9 ++++
 .../perlmods/OpenILS/Application/Vandelay.pm  | 15 ++++---
 Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm | 43 +++++++++++++++----
 3 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example
index 8d33a92778..0d60ad887b 100644
--- a/Open-ILS/examples/opensrf.xml.example
+++ b/Open-ILS/examples/opensrf.xml.example
@@ -806,6 +806,15 @@ vim:et:ts=4:sw=4:
                     <min_spare_children>1</min_spare_children>
                     <max_spare_children>5</max_spare_children>
                 </unix_config>
+                <app_settings>
+                    <databases>
+                        <!-- temporary location for MARC import files.  
+                            Files will be deleted after records are spooled.
+                            *note:  in a multi-brick environment, this will need to
+                            be on a write-able NFS share.  -->
+                        <importer>/tmp</importer>
+                    </databases>
+                </app_settings>
             </open-ils.vandelay>
 
         </apps>
diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
index d11fd2117a..3e8f1ba01f 100644
--- a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm
@@ -217,15 +217,17 @@ sub process_spool {
 
     my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint);
 	my $purpose = $data->{purpose};
-    $data = decode_base64($data->{marc});
+    my $filename = $data->{path};
 
-    $logger->info("vandelay loaded $fingerprint purpose=$purpose and ".length($data)." bytes of data");
+    unless(-r $filename) {
+        $logger->error("unable to read MARC file");
+        return -1; # make this an event XXX
+    }
 
-    my $fh;
-    open $fh, '<', \$data;
+    $logger->info("vandelay spooling $fingerprint purpose=$purpose file=$filename");
 
     my $marctype = 'USMARC'; # ?
-	my $batch = new MARC::Batch ( $marctype, $fh );
+	my $batch = new MARC::Batch ($marctype, $filename);
 	$batch->strict_off;
 
 	my $count = 0;
@@ -254,8 +256,11 @@ sub process_spool {
 	}
 
 	$e->commit;
+    unlink($filename);
+    $cache->delete_cache('vandelay_import_spool_' . $fingerprint);
 	return undef;
 }
+
 __PACKAGE__->register_method(  
 	api_name	=> "open-ils.vandelay.bib.process_spool",
 	method		=> "process_spool",
diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm b/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm
index 2014e70cc4..a65162c1d9 100644
--- a/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm
+++ b/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm
@@ -29,10 +29,13 @@ use MARC::File::XML;
 
 use MIME::Base64;
 use Digest::MD5 qw/md5_hex/;
+use OpenSRF::Utils::SettingsClient;
 
 use UNIVERSAL::require;
 
 our @formats = qw/USMARC UNIMARC XML BRE/;
+my $MAX_FILE_SIZE = 10737418240; #10G
+my $FILE_READ_SIZE = 4096;
 
 # set the bootstrap config and template include directory when
 # this module is loaded
@@ -58,18 +61,42 @@ sub spool_marc {
 
     my $data_fingerprint = '';
 	my $purpose = $cgi->param('purpose');
-	my $file = $cgi->param('marc_upload');
+	my $infile = $cgi->param('marc_upload');
 
-    if($file and -e $file) {
+	my $conf = OpenSRF::Utils::SettingsClient->new;
+	my $dir = $conf->config_value(
+        apps => 'open-ils.vandelay' => app_settings => databases => 'importer');
+
+    unless(-w $dir) {
+        $logger->error("We need some place to store our MARC files");
+	    return Apache2::Const::FORBIDDEN;
+    }
+
+    if($infile and -e $infile) {
+        my ($total_bytes, $buf, $bytes) = (0);
+	    $data_fingerprint = md5_hex(time."$$".rand());
+        my $outfile = "$dir/$data_fingerprint.mrc";
+
+        unless(open(OUTFILE, ">$outfile")) {
+            $logger->error("unable to open MARC file for writing: $@");
+	        return Apache2::Const::FORBIDDEN;
+        }
+
+        while($bytes = sysread($infile, $buf, $FILE_READ_SIZE)) {
+            $total_bytes += $bytes;
+            if($total_bytes >= $MAX_FILE_SIZE) {
+                close(OUTFILE);
+                unlink $outfile;
+	            return Apache2::Const::FORBIDDEN;
+            }
+            print OUTFILE $buf;
+        }
+
+        close(OUTFILE);
 
-	    my $data = join '', (<$file>);
-	    $data = encode_base64($data);
-    
-	    $data_fingerprint = md5_hex($data);
-    
 	    OpenSRF::Utils::Cache->new->put_cache(
 		    'vandelay_import_spool_' . $data_fingerprint,
-		    { purpose => $purpose, marc => $data }
+		    { purpose => $purpose, path => $outfile }
 	    );
     }
 
-- 
2.43.2