Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts

   1 import {Component, OnInit, AfterViewInit, Input, ViewChild, OnDestroy} from '@angular/core';
   2 import {tap} from 'rxjs/operators/tap';
   3 import {IdlObject} from '@eg/core/idl.service';
   4 import {NetService} from '@eg/core/net.service';
   5 import {EventService} from '@eg/core/event.service';
   6 import {OrgService} from '@eg/core/org.service';
   7 import {AuthService} from '@eg/core/auth.service';
   8 import {ToastService} from '@eg/share/toast/toast.service';
   9 import {ComboboxComponent,
  10     ComboboxEntry} from '@eg/share/combobox/combobox.component';
  11 import {VandelayService, VandelayImportSelection,
  12   VANDELAY_UPLOAD_PATH} from './vandelay.service';
  13 import {HttpClient, HttpRequest, HttpEventType} from '@angular/common/http';
  14 import {HttpResponse, HttpErrorResponse} from '@angular/common/http';
  15 import {ProgressInlineComponent} from '@eg/share/dialog/progress-inline.component';
  16 import {Subject} from 'rxjs/Subject';
  17 import {ServerStoreService} from '@eg/core/server-store.service';
  18
  19 const TEMPLATE_SETTING_NAME = 'eg.cat.vandelay.import.templates';
  20
  21 const TEMPLATE_ATTRS = [
  22     'recordType',
  23     'selectedBibSource',
  24     'selectedMatchSet',
  25     'mergeOnExact',
  26     'importNonMatch',
  27     'mergeOnBestMatch',
  28     'mergeOnSingleMatch',
  29     'autoOverlayAcqCopies',
  30     'selectedHoldingsProfile',
  31     'selectedMergeProfile',
  32     'selectedFallThruMergeProfile',
  33     'selectedTrashGroups',
  34     'minQualityRatio'
  35 ];
  36
  37 interface ImportOptions {
  38     session_key: string;
  39     overlay_map?: {[qrId: number]: /* breId */ number};
  40     import_no_match?: boolean;
  41     auto_overlay_exact?: boolean;
  42     auto_overlay_best_match?: boolean;
  43     auto_overlay_1match?: boolean;
  44     opp_acq_copy_overlay?: boolean;
  45     merge_profile?: any;
  46     fall_through_merge_profile?: any;
  47     strip_field_groups?: number[];
  48     match_quality_ratio: number,
  49     exit_early: boolean;
  50 }
  51
  52 @Component({
  53   templateUrl: 'import.component.html'
  54 })
  55 export class ImportComponent implements OnInit, AfterViewInit, OnDestroy {
  56
  57     recordType: string;
  58     selectedQueue: ComboboxEntry; // freetext enabled
  59
  60     // used for applying a default queue ID value when we have
  61     // a load-time queue before the queue combobox entries exist.
  62     startQueueId: number;
  63
  64     bibTrashGroups: IdlObject[];
  65     selectedTrashGroups: number[];
  66
  67     activeQueueId: number;
  68     selectedBucket: number;
  69     selectedBibSource: number;
  70     selectedMatchSet: number;
  71     selectedHoldingsProfile: number;
  72     selectedMergeProfile: number;
  73     selectedFallThruMergeProfile: number;
  74     selectedFile: File;
  75
  76     defaultMatchSet: string;
  77
  78     importNonMatching: boolean;
  79     mergeOnExact: boolean;
  80     mergeOnSingleMatch: boolean;
  81     mergeOnBestMatch: boolean;
  82     minQualityRatio: number;
  83     autoOverlayAcqCopies: boolean;
  84
  85     // True after the first upload, then remains true.
  86     showProgress: boolean;
  87
  88     // Upload in progress.
  89     isUploading: boolean;
  90
  91     // True only after successful upload
  92     uploadComplete: boolean;
  93
  94     // Upload / processsing session key
  95     // Generated by the server
  96     sessionKey: string;
  97
  98     // Optional enqueue/import tracker session name.
  99     sessionName: string;
 100
 101     selectedTemplate: string;
 102     formTemplates: {[name: string]: any};
 103     newTemplateName: string;
 104
 105     @ViewChild('fileSelector') private fileSelector;
 106     @ViewChild('uploadProgress')
 107         private uploadProgress: ProgressInlineComponent;
 108     @ViewChild('enqueueProgress')
 109         private enqueueProgress: ProgressInlineComponent;
 110     @ViewChild('importProgress')
 111         private importProgress: ProgressInlineComponent;
 112
 113     // Need these refs so values can be applied via external stimuli
 114     @ViewChild('formTemplateSelector')
 115         private formTemplateSelector: ComboboxComponent;
 116     @ViewChild('recordTypeSelector')
 117         private recordTypeSelector: ComboboxComponent;
 118     @ViewChild('bibSourceSelector')
 119         private bibSourceSelector: ComboboxComponent;
 120     @ViewChild('matchSetSelector')
 121         private matchSetSelector: ComboboxComponent;
 122     @ViewChild('holdingsProfileSelector')
 123         private holdingsProfileSelector: ComboboxComponent;
 124     @ViewChild('mergeProfileSelector')
 125         private mergeProfileSelector: ComboboxComponent;
 126     @ViewChild('fallThruMergeProfileSelector')
 127         private fallThruMergeProfileSelector: ComboboxComponent;
 128
 129     constructor(
 130         private http: HttpClient,
 131         private toast: ToastService,
 132         private evt: EventService,
 133         private net: NetService,
 134         private auth: AuthService,
 135         private org: OrgService,
 136         private store: ServerStoreService,
 137         private vandelay: VandelayService
 138     ) {
 139         this.applyDefaults();
 140     }
 141
 142     applyDefaults() {
 143         this.minQualityRatio = 0;
 144         this.selectedBibSource = 1; // default to system local
 145         this.recordType = 'bib';
 146         this.bibTrashGroups = [];
 147         this.formTemplates = {};
 148
 149         if (this.vandelay.importSelection) {
 150
 151             if (!this.vandelay.importSelection.queue) {
 152                 // Incomplete import selection, clear it.
 153                 this.vandelay.importSelection = null;
 154                 return;
 155             }
 156
 157             const queue = this.vandelay.importSelection.queue;
 158             this.recordType = queue.queue_type();
 159             this.selectedMatchSet = queue.match_set();
 160
 161             // This will be propagated to selectedQueue as a combobox
 162             // entry via the combobox
 163             this.startQueueId = queue.id();
 164
 165             if (this.recordType === 'bib') {
 166                 this.selectedBucket = queue.match_bucket();
 167                 this.selectedHoldingsProfile = queue.item_attr_def();
 168             }
 169         }
 170     }
 171
 172     ngOnInit() {}
 173
 174     ngAfterViewInit() {
 175         this.loadStartupData();
 176     }
 177
 178     ngOnDestroy() {
 179         // If we successfully completed the most recent
 180         // upload/import assume the importSelection can be cleared.
 181         if (this.uploadComplete) {
 182             this.clearSelection();
 183         }
 184     }
 185
 186     importSelection(): VandelayImportSelection {
 187         return this.vandelay.importSelection;
 188     }
 189
 190     loadStartupData(): Promise<any> {
 191         // Note displaying and manipulating a progress dialog inside
 192         // the AfterViewInit cycle leads to errors because the child
 193         // component is modifed after dirty checking.
 194
 195         const promises = [
 196             this.vandelay.getMergeProfiles(),
 197             this.vandelay.getAllQueues('bib'),
 198             this.vandelay.getAllQueues('authority'),
 199             this.vandelay.getMatchSets('bib'),
 200             this.vandelay.getMatchSets('authority'),
 201             this.vandelay.getBibBuckets(),
 202             this.vandelay.getBibSources(),
 203             this.vandelay.getItemImportDefs(),
 204             this.vandelay.getBibTrashGroups().then(
 205                 groups => this.bibTrashGroups = groups),
 206             this.org.settings(['vandelay.default_match_set']).then(
 207                 s => this.defaultMatchSet = s['vandelay.default_match_set']),
 208             this.loadTemplates()
 209         ];
 210
 211         return Promise.all(promises);
 212     }
 213
 214     loadTemplates() {
 215         this.store.getItem(TEMPLATE_SETTING_NAME).then(
 216             templates => {
 217                 this.formTemplates = templates || {};
 218
 219                 Object.keys(this.formTemplates).forEach(name => {
 220                     if (this.formTemplates[name].default) {
 221                         this.selectedTemplate = name;
 222                     }
 223                 });
 224             }
 225         );
 226     }
 227
 228     formatTemplateEntries(): ComboboxEntry[] {
 229         const entries = [];
 230
 231         Object.keys(this.formTemplates || {}).forEach(
 232             name => entries.push({id: name, label: name}));
 233
 234         return entries;
 235     }
 236
 237     // Format typeahead data sets
 238     formatEntries(etype: string): ComboboxEntry[] {
 239         const rtype = this.recordType;
 240         let list;
 241
 242         switch (etype) {
 243             case 'bibSources':
 244                 return (this.vandelay.bibSources || []).map(
 245                     s => { return {id: s.id(), label: s.source()}; });
 246
 247             case 'bibBuckets':
 248                 list = this.vandelay.bibBuckets;
 249                 break;
 250
 251             case 'allQueues':
 252                 list = this.vandelay.allQueues[rtype];
 253                 break;
 254
 255             case 'matchSets':
 256                 list = this.vandelay.matchSets[rtype];
 257                 break;
 258
 259             case 'importItemDefs':
 260                 list = this.vandelay.importItemAttrDefs;
 261                 break;
 262
 263             case 'mergeProfiles':
 264                 list = this.vandelay.mergeProfiles;
 265                 break;
 266         }
 267
 268         return (list || []).map(item => {
 269             return {id: item.id(), label: item.name()};
 270         });
 271     }
 272
 273     selectEntry($event: ComboboxEntry, etype: string) {
 274         const id = $event ? $event.id : null;
 275
 276         switch (etype) {
 277             case 'recordType':
 278                 this.recordType = id;
 279
 280             case 'bibSources':
 281                 this.selectedBibSource = id;
 282                 break;
 283
 284             case 'bibBuckets':
 285                 this.selectedBucket = id;
 286                 break;
 287
 288             case 'matchSets':
 289                 this.selectedMatchSet = id;
 290                 break;
 291
 292             case 'importItemDefs':
 293                 this.selectedHoldingsProfile = id;
 294                 break;
 295
 296             case 'mergeProfiles':
 297                 this.selectedMergeProfile = id;
 298                 break;
 299
 300             case 'FallThruMergeProfile':
 301                 this.selectedFallThruMergeProfile = id;
 302                 break;
 303         }
 304     }
 305
 306     fileSelected($event) {
 307        this.selectedFile = $event.target.files[0];
 308     }
 309
 310     // Required form data varies depending on context.
 311     hasNeededData(): boolean {
 312         if (this.vandelay.importSelection) {
 313             return this.importActionSelected();
 314         } else {
 315             return this.selectedQueue
 316                 && Boolean(this.recordType) && Boolean(this.selectedFile)
 317         }
 318     }
 319
 320     importActionSelected(): boolean {
 321         return this.importNonMatching
 322             || this.mergeOnExact
 323             || this.mergeOnSingleMatch
 324             || this.mergeOnBestMatch;
 325     }
 326
 327     // 1. create queue if necessary
 328     // 2. upload MARC file
 329     // 3. Enqueue MARC records
 330     // 4. Import records
 331     upload() {
 332         this.sessionKey = null;
 333         this.showProgress = true;
 334         this.isUploading = true;
 335         this.uploadComplete = false;
 336         this.resetProgressBars();
 337
 338         this.resolveQueue()
 339         .then(
 340             queueId => {
 341                 this.activeQueueId = queueId;
 342                 return this.uploadFile();
 343             },
 344             err => Promise.reject('queue create failed')
 345         ).then(
 346             ok => this.processSpool(),
 347             err => Promise.reject('process spool failed')
 348         ).then(
 349             ok => this.importRecords(),
 350             err => Promise.reject('import records failed')
 351         ).then(
 352             ok => {
 353                 this.isUploading = false;
 354                 this.uploadComplete = true;
 355             },
 356             err => {
 357                 console.log('file upload failed: ', err);
 358                 this.isUploading = false;
 359                 this.resetProgressBars();
 360
 361             }
 362         );
 363     }
 364
 365     resetProgressBars() {
 366         this.uploadProgress.update({value: 0, max: 1});
 367         this.enqueueProgress.update({value: 0, max: 1});
 368         this.importProgress.update({value: 0, max: 1});
 369     }
 370
 371     // Extract selected queue ID or create a new queue when requested.
 372     resolveQueue(): Promise<number> {
 373
 374         if (this.selectedQueue.freetext) {
 375         /*
 376         if (this.selectedQueue && this.selectedQueue.freetext) {
 377         */
 378             // Free text queue selector means create a new entry.
 379             // TODO: first check for name dupes
 380
 381             return this.vandelay.createQueue(
 382                 this.selectedQueue.label,
 383                 this.recordType,
 384                 this.selectedHoldingsProfile,
 385                 this.selectedMatchSet,
 386                 this.selectedBucket
 387             );
 388
 389         } else {
 390             return Promise.resolve(this.selectedQueue.id);
 391             /*
 392             var queue_id = this.startQueueId;
 393             if (this.selectedQueue) queue_id = this.selectedQueue.id;
 394             return Promise.resolve(queue_id);
 395             */
 396         }
 397     }
 398
 399     uploadFile(): Promise<any> {
 400
 401         if (this.vandelay.importSelection) {
 402             // Nothing to upload when processing pre-queued records.
 403             return Promise.resolve();
 404         }
 405
 406         const formData: FormData = new FormData();
 407
 408         formData.append('ses', this.auth.token());
 409         formData.append('marc_upload',
 410             this.selectedFile, this.selectedFile.name);
 411
 412         if (this.selectedBibSource) {
 413             formData.append('bib_source', ''+this.selectedBibSource);
 414         }
 415
 416         const req = new HttpRequest('POST', VANDELAY_UPLOAD_PATH, formData,
 417             {reportProgress: true, responseType: 'text'});
 418
 419         return this.http.request(req).pipe(tap(
 420             evt => {
 421                 if (evt.type === HttpEventType.UploadProgress) {
 422                     this.uploadProgress.update(
 423                         {value: evt.loaded, max: evt.total});
 424
 425                 } else if (evt instanceof HttpResponse) {
 426                     this.sessionKey = evt.body as string;
 427                     console.log(
 428                         'Vandelay file uploaded OK with key '+this.sessionKey);
 429                 }
 430             },
 431
 432             (err: HttpErrorResponse) => {
 433                 console.error(err);
 434                 this.toast.danger(err.error);
 435             }
 436         )).toPromise();
 437     }
 438
 439     processSpool():  Promise<any> {
 440
 441         if (this.vandelay.importSelection) {
 442             // Nothing to enqueue when processing pre-queued records
 443             return Promise.resolve();
 444         }
 445         var spoolType = this.recordType;
 446         if (this.recordType == 'authority') spoolType = 'auth'
 447
 448         const method = `open-ils.vandelay.${spoolType}.process_spool`;
 449
 450         return new Promise((resolve, reject) => {
 451             this.net.request(
 452                 'open-ils.vandelay', method,
 453                 this.auth.token(), this.sessionKey, this.activeQueueId,
 454                 null, null, this.selectedBibSource,
 455                 (this.sessionName || null), true
 456             ).subscribe(
 457                 tracker => {
 458                     const e = this.evt.parse(tracker);
 459                     if (e) { console.error(e); return reject(); }
 460
 461                     // Spooling is in progress, track the results.
 462                     this.vandelay.pollSessionTracker(tracker.id())
 463                     .subscribe(
 464                         trkr => {
 465                             this.enqueueProgress.update({
 466                                 // enqueue API only tracks actions performed
 467                                 max: null,
 468                                 value: trkr.actions_performed()
 469                             });
 470                         },
 471                         err => { console.log(err); reject(); },
 472                         () => {
 473                             this.enqueueProgress.update({max: 1, value: 1});
 474                             resolve();
 475                         }
 476                     );
 477                 }
 478             );
 479         });
 480     }
 481
 482     importRecords(): Promise<any> {
 483
 484         if (!this.importActionSelected()) {
 485             return Promise.resolve();
 486         }
 487
 488         const selection = this.vandelay.importSelection;
 489
 490         if (selection && !selection.importQueue) {
 491             return this.importRecordQueue(selection.recordIds);
 492         } else {
 493             return this.importRecordQueue();
 494         }
 495     }
 496
 497     importRecordQueue(recIds?: number[]): Promise<any> {
 498         const rtype = this.recordType === 'bib' ? 'bib' : 'auth';
 499
 500         let method = `open-ils.vandelay.${rtype}_queue.import`;
 501         const options: ImportOptions = this.compileImportOptions();
 502
 503         let target: number | number[] = this.activeQueueId;
 504         if (recIds && recIds.length) {
 505             method = `open-ils.vandelay.${rtype}_record.list.import`;
 506             target = recIds;
 507         }
 508
 509         return new Promise((resolve, reject) => {
 510             this.net.request('open-ils.vandelay',
 511                 method, this.auth.token(), target, options)
 512             .subscribe(
 513                 tracker => {
 514                     const e = this.evt.parse(tracker);
 515                     if (e) { console.error(e); return reject(); }
 516
 517                     // Spooling is in progress, track the results.
 518                     this.vandelay.pollSessionTracker(tracker.id())
 519                     .subscribe(
 520                         trkr => {
 521                             this.importProgress.update({
 522                                 max: trkr.total_actions(),
 523                                 value: trkr.actions_performed()
 524                             });
 525                         },
 526                         err => { console.log(err); reject(); },
 527                         () => {
 528                             this.importProgress.update({max: 1, value: 1});
 529                             resolve();
 530                         }
 531                     );
 532                 }
 533             );
 534         });
 535     }
 536
 537     compileImportOptions(): ImportOptions {
 538
 539         const options: ImportOptions = {
 540             session_key: this.sessionKey,
 541             import_no_match: this.importNonMatching,
 542             auto_overlay_exact: this.mergeOnExact,
 543             auto_overlay_best_match: this.mergeOnBestMatch,
 544             auto_overlay_1match: this.mergeOnSingleMatch,
 545             opp_acq_copy_overlay: this.autoOverlayAcqCopies,
 546             merge_profile: this.selectedMergeProfile,
 547             fall_through_merge_profile: this.selectedFallThruMergeProfile,
 548             strip_field_groups: this.selectedTrashGroups,
 549             match_quality_ratio: this.minQualityRatio,
 550             exit_early: true
 551         };
 552
 553         if (this.vandelay.importSelection) {
 554             options.overlay_map = this.vandelay.importSelection.overlayMap;
 555         }
 556
 557         return options;
 558     }
 559
 560     clearSelection() {
 561         this.vandelay.importSelection = null;
 562         this.startQueueId = null;
 563     }
 564
 565     openQueue() {
 566         console.log('opening queue ' + this.activeQueueId);
 567     }
 568
 569     saveTemplate() {
 570
 571         const template = {};
 572         TEMPLATE_ATTRS.forEach(key => template[key] = this[key]);
 573
 574         console.debug("Saving import profile", template);
 575
 576         this.formTemplates[this.selectedTemplate] = template;
 577         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 578     }
 579
 580     markTemplateDefault() {
 581
 582         Object.keys(this.formTemplates).forEach(
 583             name => delete this.formTemplates.default
 584         );
 585
 586         this.formTemplates[this.selectedTemplate].default = true;
 587
 588         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 589     }
 590
 591     templateSelectorChange(entry: ComboboxEntry) {
 592
 593         if (!entry) {
 594             this.selectedTemplate = '';
 595             return;
 596         }
 597
 598         this.selectedTemplate = entry.label; // label == name
 599
 600         if (entry.freetext) {
 601             // User is entering a new template name.
 602             // Nothing to apply.
 603             return;
 604         }
 605
 606         // User selected an existing template, apply it to the form.
 607
 608         const template = this.formTemplates[entry.id];
 609
 610         // Copy the template values into "this"
 611         TEMPLATE_ATTRS.forEach(key => this[key] = template[key]);
 612
 613         // Some values must be manually passed to the combobox'es
 614
 615         this.recordTypeSelector.applyEntryId(this.recordType);
 616         this.bibSourceSelector.applyEntryId(this.selectedBibSource);
 617         this.matchSetSelector.applyEntryId(this.selectedMatchSet);
 618         this.holdingsProfileSelector
 619             .applyEntryId(this.selectedHoldingsProfile);
 620         this.mergeProfileSelector.applyEntryId(this.selectedMergeProfile);
 621         this.fallThruMergeProfileSelector
 622             .applyEntryId(this.selectedFallThruMergeProfile);
 623     }
 624
 625     deleteTemplate() {
 626         delete this.formTemplates[this.selectedTemplate];
 627         this.formTemplateSelector.selected = null;
 628         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 629     }
 630 }
 631