Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts

   1 import {Component, OnInit, AfterViewInit,
   2     ViewChild, OnDestroy} from '@angular/core';
   3 import {tap} from 'rxjs/operators';
   4 import {IdlObject} from '@eg/core/idl.service';
   5 import {NetService} from '@eg/core/net.service';
   6 import {EventService} from '@eg/core/event.service';
   7 import {OrgService} from '@eg/core/org.service';
   8 import {AuthService} from '@eg/core/auth.service';
   9 import {ToastService} from '@eg/share/toast/toast.service';
  10 import {ComboboxComponent,
  11     ComboboxEntry} from '@eg/share/combobox/combobox.component';
  12 import {VandelayService, VandelayImportSelection,
  13     VANDELAY_UPLOAD_PATH} from './vandelay.service';
  14 import {HttpClient, HttpRequest, HttpEventType,
  15     HttpResponse, HttpErrorResponse} from '@angular/common/http';
  16 import {ProgressInlineComponent} from '@eg/share/dialog/progress-inline.component';
  17 import {AlertDialogComponent} from '@eg/share/dialog/alert.component';
  18 import {ServerStoreService} from '@eg/core/server-store.service';
  19
  20 const TEMPLATE_SETTING_NAME = 'eg.cat.vandelay.import.templates';
  21
  22 const TEMPLATE_ATTRS = [
  23     'recordType',
  24     'selectedBibSource',
  25     'selectedMatchSet',
  26     'mergeOnExact',
  27     'importNonMatching',
  28     'mergeOnBestMatch',
  29     'mergeOnSingleMatch',
  30     'autoOverlayAcqCopies',
  31     'autoOverlayOnOrderCopies',
  32     'autoOverlayOrgUnitCopies',
  33     'selectedHoldingsProfile',
  34     'selectedMergeProfile',
  35     'selectedFallThruMergeProfile',
  36     'selectedTrashGroups',
  37     'minQualityRatio'
  38 ];
  39
  40 interface ImportOptions {
  41     session_key: string;
  42     overlay_map?: {[qrId: number]: /* breId */ number};
  43     import_no_match?: boolean;
  44     auto_overlay_exact?: boolean;
  45     auto_overlay_best_match?: boolean;
  46     auto_overlay_1match?: boolean;
  47     opp_acq_copy_overlay?: boolean;
  48     opp_oo_cat_copy_overlay?: boolean;
  49     auto_overlay_org_unit_copies?: boolean;
  50     merge_profile?: any;
  51     fall_through_merge_profile?: any;
  52     strip_field_groups?: number[];
  53     match_quality_ratio: number;
  54     exit_early: boolean;
  55 }
  56
  57 @Component({
  58     templateUrl: 'import.component.html'
  59 })
  60 export class ImportComponent implements OnInit, AfterViewInit, OnDestroy {
  61
  62     recordType: string;
  63     selectedQueue: ComboboxEntry; // freetext enabled
  64
  65     // used for applying a default queue ID value when we have
  66     // a load-time queue before the queue combobox entries exist.
  67     startQueueId: number;
  68
  69     bibTrashGroups: IdlObject[];
  70     selectedTrashGroups: number[];
  71
  72     activeQueueId: number;
  73     selectedBucket: number;
  74     selectedBibSource: number;
  75     selectedMatchSet: number;
  76     selectedHoldingsProfile: number;
  77     selectedMergeProfile: number;
  78     selectedFallThruMergeProfile: number;
  79     selectedFile: File;
  80
  81     defaultMatchSet: string;
  82
  83     importNonMatching: boolean;
  84     mergeOnExact: boolean;
  85     mergeOnSingleMatch: boolean;
  86     mergeOnBestMatch: boolean;
  87     minQualityRatio: number;
  88     autoOverlayAcqCopies: boolean;
  89     autoOverlayOnOrderCopies: boolean;
  90     autoOverlayOrgUnitCopies: boolean;
  91
  92     // True after the first upload, then remains true.
  93     showProgress: boolean;
  94
  95     // Upload in progress.
  96     isUploading: boolean;
  97
  98     // True only after successful upload
  99     uploadComplete: boolean;
 100
 101     // Upload / processsing session key
 102     // Generated by the server
 103     sessionKey: string;
 104
 105     // Optional enqueue/import tracker session name.
 106     sessionName: string;
 107
 108     selectedTemplate: string;
 109     formTemplates: {[name: string]: any};
 110     newTemplateName: string;
 111
 112     @ViewChild('fileSelector', { static: false }) private fileSelector;
 113     @ViewChild('uploadProgress', { static: true })
 114     private uploadProgress: ProgressInlineComponent;
 115     @ViewChild('enqueueProgress', { static: true })
 116     private enqueueProgress: ProgressInlineComponent;
 117     @ViewChild('importProgress', { static: true })
 118     private importProgress: ProgressInlineComponent;
 119
 120     // Need these refs so values can be applied via external stimuli
 121     @ViewChild('formTemplateSelector', { static: true })
 122     private formTemplateSelector: ComboboxComponent;
 123     @ViewChild('recordTypeSelector', { static: true })
 124     private recordTypeSelector: ComboboxComponent;
 125     @ViewChild('bibSourceSelector', { static: true })
 126     private bibSourceSelector: ComboboxComponent;
 127     @ViewChild('matchSetSelector', { static: true })
 128     private matchSetSelector: ComboboxComponent;
 129     @ViewChild('holdingsProfileSelector', { static: true })
 130     private holdingsProfileSelector: ComboboxComponent;
 131     @ViewChild('mergeProfileSelector', { static: true })
 132     private mergeProfileSelector: ComboboxComponent;
 133     @ViewChild('fallThruMergeProfileSelector', { static: true })
 134     private fallThruMergeProfileSelector: ComboboxComponent;
 135     @ViewChild('queueSelector') private queueSelector: ComboboxComponent;
 136     @ViewChild('bucketSelector') private bucketSelector: ComboboxComponent;
 137
 138     @ViewChild('dupeQueueAlert', { static: true })
 139     private dupeQueueAlert: AlertDialogComponent;
 140
 141     constructor(
 142         private http: HttpClient,
 143         private toast: ToastService,
 144         private evt: EventService,
 145         private net: NetService,
 146         private auth: AuthService,
 147         private org: OrgService,
 148         private store: ServerStoreService,
 149         private vandelay: VandelayService
 150     ) {
 151     }
 152
 153     applyDefaults() {
 154         this.minQualityRatio = 0;
 155         this.selectedBibSource = 1; // default to system local
 156         this.recordType = 'bib';
 157         this.bibTrashGroups = [];
 158         this.formTemplates = {};
 159
 160         if (this.vandelay.importSelection) {
 161             // Apply start-id values to our comboboxes based on our
 162             // import selection values.
 163             const queue = this.vandelay.importSelection.queue;
 164
 165             if (!queue) {
 166                 // Incomplete import selection, clear it.
 167                 this.vandelay.importSelection = null;
 168                 return;
 169             }
 170
 171             this.recordType = queue.queue_type();
 172             this.startQueueId = queue.id();
 173             this.selectedMatchSet = queue.match_set();
 174
 175             if (this.recordType === 'bib') {
 176                 this.selectedBucket = queue.match_bucket();
 177                 this.selectedHoldingsProfile = queue.item_attr_def();
 178             }
 179         }
 180     }
 181
 182     applyImportSelection() {
 183
 184         // Depending on when the comboboxes are rendered, the start-id values
 185         // applied in applyDefaults() may not have any affect.  Ensure we
 186         // get the initial values we want by manually setting selectedIds
 187         // to the rendered combobox as well.
 188
 189         const queue = this.vandelay.importSelection.queue;
 190         if (!queue) {
 191             // Incomplete import selection, clear it.
 192             this.vandelay.importSelection = null;
 193             return;
 194         }
 195
 196         if (this.queueSelector) {
 197             this.queueSelector.selectedId = this.startQueueId;
 198             this.selectedQueue = {id: queue.id(), label: queue.name()};
 199         }
 200
 201         if (this.matchSetSelector) {
 202             this.matchSetSelector.selectedId = this.selectedMatchSet;
 203         }
 204
 205         if (this.recordType === 'bib') {
 206             if (this.holdingsProfileSelector) {
 207                 this.holdingsProfileSelector.selectedId = this.selectedHoldingsProfile;
 208             }
 209             if (this.bucketSelector) {
 210                 this.bucketSelector.selectedId = this.selectedBucket;
 211             }
 212         }
 213     }
 214
 215     ngOnInit() {
 216         this.applyDefaults();
 217     }
 218
 219     ngAfterViewInit() {
 220         if (this.vandelay.importSelection) {
 221             // setTimeout() is not required here, but it helps to
 222             // avoid expression-changed-after-checking console messages.
 223             setTimeout(() => this.applyImportSelection());
 224         }
 225         this.loadStartupData();
 226     }
 227
 228     ngOnDestroy() {
 229         // Always clear the import selection when navigating away from
 230         // the import page.
 231         this.clearSelection();
 232     }
 233
 234     importSelection(): VandelayImportSelection {
 235         return this.vandelay.importSelection;
 236     }
 237
 238     loadStartupData(): Promise<any> {
 239         // Note displaying and manipulating a progress dialog inside
 240         // the AfterViewInit cycle leads to errors because the child
 241         // component is modifed after dirty checking.
 242
 243         const promises = [
 244             this.vandelay.getMergeProfiles(),
 245             this.vandelay.getAllQueues('bib'),
 246             this.vandelay.getAllQueues('authority'),
 247             this.vandelay.getMatchSets('bib'),
 248             this.vandelay.getMatchSets('authority'),
 249             this.vandelay.getBibBuckets(),
 250             this.vandelay.getBibSources(),
 251             this.vandelay.getItemImportDefs(),
 252             this.vandelay.getBibTrashGroups().then(
 253                 groups => this.bibTrashGroups = groups),
 254             this.org.settings(['vandelay.default_match_set']).then(
 255                 s => this.defaultMatchSet = s['vandelay.default_match_set']),
 256             this.loadTemplates()
 257         ];
 258
 259         return Promise.all(promises);
 260     }
 261
 262     loadTemplates() {
 263         this.store.getItem(TEMPLATE_SETTING_NAME).then(
 264             templates => {
 265                 this.formTemplates = templates || {};
 266
 267                 Object.keys(this.formTemplates).forEach(name => {
 268                     if (this.formTemplates[name].default) {
 269                         this.selectedTemplate = name;
 270                     }
 271                 });
 272             }
 273         );
 274     }
 275
 276     formatTemplateEntries(): ComboboxEntry[] {
 277         const entries = [];
 278
 279         Object.keys(this.formTemplates || {}).forEach(
 280             name => entries.push({id: name, label: name}));
 281
 282         return entries;
 283     }
 284
 285     // Format typeahead data sets
 286     formatEntries(etype: string): ComboboxEntry[] {
 287         const rtype = this.recordType;
 288         let list;
 289
 290         switch (etype) {
 291             case 'bibSources':
 292                 return (this.vandelay.bibSources || []).map(
 293                     s => {
 294                         return {id: s.id(), label: s.source()};
 295                     });
 296
 297             case 'bibBuckets':
 298                 list = this.vandelay.bibBuckets;
 299                 break;
 300
 301             case 'activeQueues':
 302                 list = (this.vandelay.allQueues[rtype] || [])
 303                     .filter(q => q.complete() === 'f');
 304                 break;
 305
 306             case 'matchSets':
 307                 list = this.vandelay.matchSets[rtype];
 308                 break;
 309
 310             case 'importItemDefs':
 311                 list = this.vandelay.importItemAttrDefs;
 312                 break;
 313
 314             case 'mergeProfiles':
 315                 list = this.vandelay.mergeProfiles;
 316                 break;
 317         }
 318
 319         return (list || []).map(item => {
 320             return {id: item.id(), label: item.name()};
 321         });
 322     }
 323
 324     selectEntry($event: ComboboxEntry, etype: string) {
 325         const id = $event ? $event.id : null;
 326
 327         switch (etype) {
 328             case 'recordType':
 329                 this.recordType = id;
 330                 break;
 331
 332             case 'bibSources':
 333                 this.selectedBibSource = id;
 334                 break;
 335
 336             case 'bibBuckets':
 337                 this.selectedBucket = id;
 338                 break;
 339
 340             case 'matchSets':
 341                 this.selectedMatchSet = id;
 342                 break;
 343
 344             case 'importItemDefs':
 345                 this.selectedHoldingsProfile = id;
 346                 break;
 347
 348             case 'mergeProfiles':
 349                 this.selectedMergeProfile = id;
 350                 break;
 351
 352             case 'FallThruMergeProfile':
 353                 this.selectedFallThruMergeProfile = id;
 354                 break;
 355         }
 356     }
 357
 358     fileSelected($event) {
 359         this.selectedFile = $event.target.files[0];
 360     }
 361
 362     // Required form data varies depending on context.
 363     hasNeededData(): boolean {
 364         if (this.vandelay.importSelection) {
 365             return this.importActionSelected();
 366         } else {
 367             return this.selectedQueue &&
 368                 Boolean(this.recordType) && Boolean(this.selectedFile);
 369         }
 370     }
 371
 372     importActionSelected(): boolean {
 373         return this.importNonMatching
 374             || this.mergeOnExact
 375             || this.mergeOnSingleMatch
 376             || this.mergeOnBestMatch;
 377     }
 378
 379     // 1. create queue if necessary
 380     // 2. upload MARC file
 381     // 3. Enqueue MARC records
 382     // 4. Import records
 383     upload() {
 384         this.sessionKey = null;
 385         this.showProgress = true;
 386         this.isUploading = true;
 387         this.uploadComplete = false;
 388         this.resetProgressBars();
 389
 390         this.resolveQueue()
 391             .then(
 392                 queueId => {
 393                     this.activeQueueId = queueId;
 394                     return this.uploadFile();
 395                 },
 396                 err => Promise.reject('queue create failed')
 397             ).then(
 398                 ok => this.processSpool(),
 399                 err => Promise.reject('process spool failed')
 400             ).then(
 401                 ok => this.importRecords(),
 402                 err => Promise.reject('import records failed')
 403             ).then(
 404                 ok => {
 405                     this.isUploading = false;
 406                     this.uploadComplete = true;
 407                 },
 408                 err => {
 409                     console.log('file upload failed: ', err);
 410                     this.isUploading = false;
 411                     this.resetProgressBars();
 412
 413                 }
 414             );
 415     }
 416
 417     resetProgressBars() {
 418         this.uploadProgress.update({value: 0, max: 1});
 419         this.enqueueProgress.update({value: 0, max: 1});
 420         this.importProgress.update({value: 0, max: 1});
 421     }
 422
 423     // Extract selected queue ID or create a new queue when requested.
 424     resolveQueue(): Promise<number> {
 425
 426         if (this.selectedQueue.freetext) {
 427             // Free text queue selector means create a new entry.
 428             // TODO: first check for name dupes
 429
 430             return this.vandelay.createQueue(
 431                 this.selectedQueue.label,
 432                 this.recordType,
 433                 this.selectedHoldingsProfile,
 434                 this.selectedMatchSet,
 435                 this.selectedBucket
 436             ).then(
 437                 id => id,
 438                 err => {
 439                     const evt = this.evt.parse(err);
 440                     if (evt) {
 441                         if (evt.textcode.match(/QUEUE_EXISTS/)) {
 442                             this.dupeQueueAlert.open();
 443                         } else {
 444                             alert(evt); // server error
 445                         }
 446                     }
 447
 448                     return Promise.reject('Queue Create Failed');
 449                 }
 450             );
 451         } else {
 452             return Promise.resolve(this.selectedQueue.id);
 453         }
 454     }
 455
 456     uploadFile(): Promise<any> {
 457
 458         if (this.vandelay.importSelection) {
 459             // Nothing to upload when processing pre-queued records.
 460             return Promise.resolve();
 461         }
 462
 463         const formData: FormData = new FormData();
 464
 465         formData.append('ses', this.auth.token());
 466         formData.append('marc_upload',
 467             this.selectedFile, this.selectedFile.name);
 468
 469         if (this.selectedBibSource) {
 470             formData.append('bib_source', '' + this.selectedBibSource);
 471         }
 472
 473         const req = new HttpRequest('POST', VANDELAY_UPLOAD_PATH, formData,
 474             {reportProgress: true, responseType: 'text'});
 475
 476         return this.http.request(req).pipe(tap(
 477             evt => {
 478                 if (evt.type === HttpEventType.UploadProgress) {
 479                     this.uploadProgress.update(
 480                         {value: evt.loaded, max: evt.total});
 481
 482                 } else if (evt instanceof HttpResponse) {
 483                     this.sessionKey = evt.body as string;
 484                     console.log(
 485                         'Vandelay file uploaded OK with key ' + this.sessionKey);
 486                 }
 487             },
 488
 489             // eslint-disable-next-line rxjs/no-implicit-any-catch
 490             (err: HttpErrorResponse) => {
 491                 console.error(err);
 492                 this.toast.danger(err.error);
 493             }
 494         )).toPromise();
 495     }
 496
 497     processSpool():  Promise<any> {
 498
 499         if (this.vandelay.importSelection) {
 500             // Nothing to enqueue when processing pre-queued records
 501             return Promise.resolve();
 502         }
 503
 504         let spoolType = this.recordType;
 505         if (this.recordType === 'authority') {
 506             spoolType = 'auth';
 507         }
 508
 509         const method = `open-ils.vandelay.${spoolType}.process_spool`;
 510
 511         return new Promise((resolve, reject) => {
 512             this.net.request(
 513                 'open-ils.vandelay', method,
 514                 this.auth.token(), this.sessionKey, this.activeQueueId,
 515                 null, null, this.selectedBibSource,
 516                 (this.sessionName || null), true
 517             ).subscribe(
 518                 tracker => {
 519                     const e = this.evt.parse(tracker);
 520                     if (e) { console.error(e); return reject(); }
 521
 522                     // Spooling is in progress, track the results.
 523                     this.vandelay.pollSessionTracker(tracker.id())
 524                         // eslint-disable-next-line rxjs/no-nested-subscribe
 525                         .subscribe(
 526                             trkr => {
 527                                 this.enqueueProgress.update({
 528                                 // enqueue API only tracks actions performed
 529                                     max: null,
 530                                     value: trkr.actions_performed()
 531                                 });
 532                             },
 533                             (err: unknown) => { console.log(err); reject(); },
 534                             () => {
 535                                 this.enqueueProgress.update({max: 1, value: 1});
 536                                 resolve(null);
 537                             }
 538                         );
 539                 }
 540             );
 541         });
 542     }
 543
 544     importRecords(): Promise<any> {
 545
 546         if (!this.importActionSelected()) {
 547             return Promise.resolve();
 548         }
 549
 550         const selection = this.vandelay.importSelection;
 551
 552         if (selection && !selection.importQueue) {
 553             return this.importRecordQueue(selection.recordIds);
 554         } else {
 555             return this.importRecordQueue();
 556         }
 557     }
 558
 559     importRecordQueue(recIds?: number[]): Promise<any> {
 560         const rtype = this.recordType === 'bib' ? 'bib' : 'auth';
 561
 562         let method = `open-ils.vandelay.${rtype}_queue.import`;
 563         const options: ImportOptions = this.compileImportOptions();
 564
 565         let target: number | number[] = this.activeQueueId;
 566         if (recIds && recIds.length) {
 567             method = `open-ils.vandelay.${rtype}_record.list.import`;
 568             target = recIds;
 569         }
 570
 571         return new Promise((resolve, reject) => {
 572             this.net.request('open-ils.vandelay',
 573                 method, this.auth.token(), target, options)
 574                 .subscribe(
 575                     tracker => {
 576                         const e = this.evt.parse(tracker);
 577                         if (e) { console.error(e); return reject(); }
 578
 579                         // Spooling is in progress, track the results.
 580                         this.vandelay.pollSessionTracker(tracker.id())
 581                             // eslint-disable-next-line rxjs/no-nested-subscribe
 582                             .subscribe(
 583                                 trkr => {
 584                                     this.importProgress.update({
 585                                         max: trkr.total_actions(),
 586                                         value: trkr.actions_performed()
 587                                     });
 588                                 },
 589                                 (err: unknown) => { console.log(err); reject(); },
 590                                 () => {
 591                                     this.importProgress.update({max: 1, value: 1});
 592                                     resolve(null);
 593                                 }
 594                             );
 595                     }
 596                 );
 597         });
 598     }
 599
 600     compileImportOptions(): ImportOptions {
 601
 602         const options: ImportOptions = {
 603             session_key: this.sessionKey,
 604             import_no_match: this.importNonMatching,
 605             auto_overlay_exact: this.mergeOnExact,
 606             auto_overlay_best_match: this.mergeOnBestMatch,
 607             auto_overlay_1match: this.mergeOnSingleMatch,
 608             opp_acq_copy_overlay: this.autoOverlayAcqCopies,
 609             opp_oo_cat_copy_overlay: this.autoOverlayOnOrderCopies,
 610             auto_overlay_org_unit_copies: this.autoOverlayOrgUnitCopies,
 611             merge_profile: this.selectedMergeProfile,
 612             fall_through_merge_profile: this.selectedFallThruMergeProfile,
 613             strip_field_groups: this.selectedTrashGroups,
 614             match_quality_ratio: this.minQualityRatio,
 615             exit_early: true
 616         };
 617
 618         if (this.vandelay.importSelection) {
 619             options.overlay_map = this.vandelay.importSelection.overlayMap;
 620         }
 621
 622         return options;
 623     }
 624
 625     clearSelection() {
 626         this.vandelay.importSelection = null;
 627         this.startQueueId = null;
 628     }
 629
 630     openQueue() {
 631         console.log('opening queue ' + this.activeQueueId);
 632     }
 633
 634     saveTemplate() {
 635
 636         const template = {};
 637         TEMPLATE_ATTRS.forEach(key => template[key] = this[key]);
 638
 639         console.debug('Saving import profile', template);
 640
 641         this.formTemplates[this.selectedTemplate] = template;
 642         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 643     }
 644
 645     markTemplateDefault() {
 646
 647         Object.keys(this.formTemplates).forEach(
 648             name => delete this.formTemplates.default
 649         );
 650
 651         this.formTemplates[this.selectedTemplate].default = true;
 652
 653         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 654     }
 655
 656     templateSelectorChange(entry: ComboboxEntry) {
 657
 658         if (!entry) {
 659             this.selectedTemplate = '';
 660             return;
 661         }
 662
 663         this.selectedTemplate = entry.label; // label == name
 664
 665         if (entry.freetext) {
 666             // User is entering a new template name.
 667             // Nothing to apply.
 668             return;
 669         }
 670
 671         // User selected an existing template, apply it to the form.
 672
 673         const template = this.formTemplates[entry.id];
 674
 675         // Copy the template values into "this"
 676         TEMPLATE_ATTRS.forEach(key => this[key] = template[key]);
 677
 678         // Some values must be manually passed to the combobox'es
 679
 680         this.recordTypeSelector.applyEntryId(this.recordType);
 681         this.bibSourceSelector.applyEntryId(this.selectedBibSource);
 682         this.matchSetSelector.applyEntryId(this.selectedMatchSet);
 683         this.holdingsProfileSelector
 684             .applyEntryId(this.selectedHoldingsProfile);
 685         this.mergeProfileSelector.applyEntryId(this.selectedMergeProfile);
 686         this.fallThruMergeProfileSelector
 687             .applyEntryId(this.selectedFallThruMergeProfile);
 688     }
 689
 690     deleteTemplate() {
 691         delete this.formTemplates[this.selectedTemplate];
 692         this.formTemplateSelector.selected = null;
 693         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 694     }
 695 }
 696