From c756f6bebb8bcff1de84f9243aa640c5a3ac06dd Mon Sep 17 00:00:00 2001 From: Robert Soulliere Date: Tue, 8 Feb 2011 14:18:20 -0500 Subject: [PATCH] change name of migratingdata to migratingdata_2.0.xml --- ...igratingdata.xml => migratingdata_1.6.xml} | 215 ++++++++++++++++-- 1.6/root.xml | 2 +- 2 files changed, 195 insertions(+), 22 deletions(-) rename 1.6/admin/{migratingdata.xml => migratingdata_1.6.xml} (72%) diff --git a/1.6/admin/migratingdata.xml b/1.6/admin/migratingdata_1.6.xml similarity index 72% rename from 1.6/admin/migratingdata.xml rename to 1.6/admin/migratingdata_1.6.xml index af49df8cb2..b32c0086a1 100644 --- a/1.6/admin/migratingdata.xml +++ b/1.6/admin/migratingdata_1.6.xml @@ -127,10 +127,12 @@ http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
- Migrating Bibliographic Records Using the ESI MigUration Tools - migratingMigration Tools - The following procedure explains how to migrate bibliographic records from marc records into Evergreen. It does not cover exporting records from specific proprietary ILS - systems. For assistance with exporting records from your current system please refer to the manuals for your system or you might try to ask for help from the Evergreen community. + Migrating Bibliographic Records Using the ESI Migration Tools + migratingbibliograohic records + The following procedure explains how to migrate bibliographic records from marc records into Evergreen. This is a general guide and will need to be adjusted for your + specific environment. It does not cover exporting records from specific proprietary ILS + systems. For assistance with exporting records from your current system please refer to the manuals for your system or you might try to ask for help from the + Evergreen community. @@ -150,6 +152,22 @@ make install + + Add environmental variables for migration and import tools. These paths must point to: + + the import perl scripts bundled with Evergreen + the folder where you extracted the migration tools + the location of the Equinox-Migration perl modules + the location of the Evergreen perl modules (e.g. perl5) + + + + +export PATH=[path to Evergreen]/Open-ILS/src/extras/import:/[path to migration-tools]/migration-tools:$PATH:. +export PERL5LIB=/openils/lib/perl5:/[path to migration-tools directory]/migration-tools/Equinox-Migration/lib + + + Dump marc records into MARCXML using yaz-marcdump @@ -164,7 +182,6 @@ yaz-marcdump -f MARC-8 -t UTF-8 -o marcxml imported_marc_records.mrc >> imported Test validity of XML file using xmllint - @@ -190,10 +207,10 @@ marc_cleanup --marcfile=imported_marc_records.xml --fullauto [--renumber-from #] fingerprinter -o incumbent.fp -x incumbent.ex clean.marc.xml - + + fingerprinter is used for deduplification of the incumbent records. The option specifies the output file and the option is used to specify the error output file. - Create a fingerprinter file for existing Evergreen bibliographic records using the fingerprinter utility if you @@ -222,6 +239,7 @@ match_fingerprints [-t start id] -o records.merge dedupe.fp extract_loadset -l 1 -i clean.marc.xml -o merged.xml records.merge + Extract all of the currently used TCN's an generate the .bre and .ingest files to prepare for the bibliographic record load. @@ -268,7 +286,7 @@ parallel_pg_loader.pl \ load_pg_loader-output -psql -U evergreen < create_metabib.sql > log.create_metabib +psql -U evergreen < ~/Ever*/Open-ILS/src/extras/import/quick_metarecord_map.sql > log.create_metabib ]]> @@ -277,10 +295,10 @@ psql -U evergreen < create_metabib.sql > log.create_metabib Extract holdings from marc records for importing copies into Evergreen using the extract_holdings utility. -extract_holdings --marcfile=marc.clean.xml --holding 999 --copyid 999i --map holdings.map +extract_holdings --marcfile=clean.marc.xml --holding 999 --copyid 999i --map holdings.map - This command would extract holdings based on the 949 datafield in the marc records. The copy id is generated from the subfile b in the 949 datafield. You may + This command would extract holdings based on the 949 datafield in the marc records. The copy id is generated from the subfile i in the 999 datafield. You may need to adjust these options based on the field used for holdings informatiom in your marc records. The option holdings.map refers to a file to be used for mapping subfields to the holdings data you would like extracted. Here is an example based on mapping holdings data to the 999 data field: @@ -292,7 +310,7 @@ owning_lib 999 m circ_modifier 999 t ]]> - Running the extract holdings script should produce the sql file HOLDINGS.pg similar to: + Running the extract holdings script should produce an sql script HOLDINGS.pg similar to: BEGIN; @@ -303,15 +321,26 @@ egid, hseq, l_call_num, l_barcode, l_location, l_owning_lib, l_circ_modifier, ... - This file can be used for importing holdings into Evergreen. the egid is a critical column. It is used to link the volume and copy to - the bibliographic record. Please refer to for the steps to import your holdings into Evergreen. - + Edit the holdings.pg sql script like so: + +BEGIN; + +TRUNCATE TABLE staging_items; + +INSERT INTO staging_items (egid, hseq, l_call_num, l_barcode, l_location, l_owning_lib, l_circ_modifier FROM stdin; +40 0 HD3616.K853 U54 1997 30731100751928 STACKS FENNELL BOOK +41 1 HV6548.C3 S984 1998 30731100826613 STACKS FENNELL BOOK +41 2 HV6548.C3 S984 1998 30731100804958 STACKS BRANTFORD BOOK +COMMIT; + + This file can be used for importing holdings into Evergreen. the egid is a critical column. It is used to link the volume and + copy to the bibliographic record. Please refer to for the steps to import your holdings into Evergreen.
- Adding Copies to Bibliographic Records + Adding Copies to Bibliographic Recordsmigratingholdings Before bibliographic records can be found in an OPAC search copies will need to be created. It is very important to understand how various tables related to each other in regards to holdings maintenance. The following procedure will guide you through the process of populating Evergreen with volumes and copies. This is a very simple example. The SQL queries may need to be adjusted @@ -322,7 +351,8 @@ egid, hseq, l_call_num, l_barcode, l_location, l_owning_lib, l_circ_modifier, CREATE TABLE staging_items ( callnum text, -- call number label - bibkey int, -- biblio.record_entry_id + hseq int, -- call number label + egid int, -- biblio.record_entry_id createdate date, location text, barcode text, @@ -332,10 +362,10 @@ CREATE TABLE staging_items ( - Login to the Evergreen using psql and run the following COPY command to copy the items generated from extract_holdings utility: - -COPY staging_items (bibkey, id, callnum, barcode, location, owning_lib, item_type) FROM 'HOLDINGS.pg'; - + Import the items using the HOLDINGS.pg SQL script created using the extract_holdings utility. + +psql -U evergreen -f HOLDINGS.pg evergreen + the file HOLDINGS.pg and/or the COPY query may need to be adjusted for your particular circumstances. @@ -409,7 +439,7 @@ SELECT DISTINCT ou.id AS circ_lib,
Migrating Patron Data - migratingimporting patrons + migratingpatrons This section will explain the task of migrating your patron data from comma delimited filescomma delimited files into Evergreen. It does not deal with the process of exporting from the non-Evergreen @@ -641,4 +671,147 @@ COMMIT;
+
+ Exporting Bibliographic Records into MARC files + migratingexporting bibliographic recordsMARC + The following procedure explains how to export Evergreen bibliographic records into MARC files using the marc_export support script. All steps should be performed by the + opensrf user from your Evergreen server. + + + Create a text file list of the Bibliographic record ids you would like to export from Evergreen. One way to do this is using SQL: + +SELECT DISTINCT bre.id FROM biblio.record_entry AS bre + JOIN asset.call_number AS acn ON acn.record-bre.id + WHERE bre.deleted='false' and ownling_lib=101 \g /home/opensrf/records.txt; + + This query will create a file called ids.txt containing a column of distinct ids of items owned by the organizational unit with the + id
101. + + + Navigate to the support-scripts folder + +cd /home/opensrf/Evergreen-ILS*/Open-ILS/src/support-scripts/ + + + + Run marc_export, using the id file you created in step 1 to define which files to export. + +cat /home/opensrf/records.txt | ./marc_export -i -c /openils/conf/opensrf_core.xml -x /openils/conf/fm_IDL.xml -f XML --timeout 5 > exported_files.xml + + + The example above export the records into MARCXML format. + + For help or for more options when running marc_export, run marc_export with the option: + +./marc_export -h + + + + + + Processing time for exporting records will depond on several factors such as the number of records you are exporting. It is recommended that you divide the + export id files (records.txt) into manageable number of records if you are exporting a large number of records. + + +
+
+ Importing Authority Records + migratingimporting authority records + The following procedure explains how to export Evergreen bibliographic records into MARC files using the marc_export support script. All steps should be performed by the + opensrf user from your Evergreen server. + + Importing Authority Records from Command Line + The major advantages of the command line approach are its speed and its convenience for system administrators who can perform bulk loads of authority records in a + controlled environment. + + + + Run marc2are.pl against the authority records, specifying the user name, password, MARC type (USMARC or XML). Use + STDOUT redirection + to either pipe the output directly into the next command or into an output file for inspection. For example, to process a set of authority records + named auth_small.xml using the default user name and password and directing the output into a file named auth.are: + +cd Open-ILS/src/extras/import/ +perl marc2are.pl --user admin --pass open-ils auth_small.xml > auth.are + + + + + + Run direct_ingest.pl to ingest records. + +perl direct_ingest.pl -a auth.are > ~/auth.ingest + + + + Run pg_loader.pl to generate the SQL necessary for importing the authority records into your system. + +cd Open-ILS/src/extras/import/ + perl pg_loader.pl-or are -or afr -a afr --output=auth < ~/auth.ingest + + + + + Load the authority records from the SQL file that you generated in the last step into your Evergreen database using the psql tool. Assuming the default user + name, host name, and database name for an Evergreen instance, that command looks like: + +psql -U evergreen -h localhost -d evergreen -f auth.sql + + + + + + + Importing authority records using the MARC Batch Import/Export interface from the Staff Client + Good for loading batches of up to 5,000 records (roughly) at a time, the major advantages to importing authority records using the MARC Batch Import/Export interface are + that it does not require command-line or direct database access – good for both security in that it minimizes the number of people who need this access and for spreading the + effort around to others in the library – and it does most of the work (for example, figuring out whether the batch of records is in XML or USMARC format) for you. + To import a set of MARC authority records from the MARC Batch Import/Export interface: + + + From the Evergreen staff client, select CatalogingMARC Batch Import/Export. + The Evergreen MARC File Upload screen opens, with Import Records as the highlighted tab. + + + From the Bibliographic records drop-down menu, select Authority records. + + + + Enter a name for the queue (batch import job) in the Create a new upload queue field. + + + + Select the Auto-Import Non-Colliding Records checkbox. + + + + Click the Browse… button to select the file of MARC authorities to import. + + + + Click the Upload button to begin importing the records. The screen displays Uploading… + Processing… to show that the records are being transferred to the server, then displays a progress bar to show the actual import + progress. When the staff client displays the progress bar, you can disconnect your staff client safely. Very large batches of records might time out at this + stage. + + + + Once the import is finished, the staff client displays the results of the import process. You can manually display the import progress by selecting + the Inspect Queue tab of the MARC Batch Import/Export interface and selecting the queue name. By default, the staff client does not + display records that were imported successfully; it only shows records that conflicted with existing entries in the database. The screen shows the overall + status of the import process in the top right-hand corner, with the Total and Imported number of records for the + queue. + + + + + +
+ + + + + + + diff --git a/1.6/root.xml b/1.6/root.xml index e04478d05b..4445efd887 100755 --- a/1.6/root.xml +++ b/1.6/root.xml @@ -74,7 +74,7 @@ - + -- 2.43.2