Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts

   1 import {Component, OnInit, AfterViewInit, Input, ViewChild, OnDestroy} from '@angular/core';
   2 import {tap} from 'rxjs/operators/tap';
   3 import {IdlObject} from '@eg/core/idl.service';
   4 import {NetService} from '@eg/core/net.service';
   5 import {EventService} from '@eg/core/event.service';
   6 import {OrgService} from '@eg/core/org.service';
   7 import {AuthService} from '@eg/core/auth.service';
   8 import {ToastService} from '@eg/share/toast/toast.service';
   9 import {ComboboxComponent,
  10     ComboboxEntry} from '@eg/share/combobox/combobox.component';
  11 import {VandelayService, VandelayImportSelection,
  12   VANDELAY_UPLOAD_PATH} from './vandelay.service';
  13 import {HttpClient, HttpRequest, HttpEventType} from '@angular/common/http';
  14 import {HttpResponse, HttpErrorResponse} from '@angular/common/http';
  15 import {ProgressInlineComponent} from '@eg/share/dialog/progress-inline.component';
  16 import {AlertDialogComponent} from '@eg/share/dialog/alert.component';
  17 import {Subject} from 'rxjs/Subject';
  18 import {ServerStoreService} from '@eg/core/server-store.service';
  19
  20 const TEMPLATE_SETTING_NAME = 'eg.cat.vandelay.import.templates';
  21
  22 const TEMPLATE_ATTRS = [
  23     'recordType',
  24     'selectedBibSource',
  25     'selectedMatchSet',
  26     'mergeOnExact',
  27     'importNonMatch',
  28     'mergeOnBestMatch',
  29     'mergeOnSingleMatch',
  30     'autoOverlayAcqCopies',
  31     'selectedHoldingsProfile',
  32     'selectedMergeProfile',
  33     'selectedFallThruMergeProfile',
  34     'selectedTrashGroups',
  35     'minQualityRatio'
  36 ];
  37
  38 interface ImportOptions {
  39     session_key: string;
  40     overlay_map?: {[qrId: number]: /* breId */ number};
  41     import_no_match?: boolean;
  42     auto_overlay_exact?: boolean;
  43     auto_overlay_best_match?: boolean;
  44     auto_overlay_1match?: boolean;
  45     opp_acq_copy_overlay?: boolean;
  46     merge_profile?: any;
  47     fall_through_merge_profile?: any;
  48     strip_field_groups?: number[];
  49     match_quality_ratio: number,
  50     exit_early: boolean;
  51 }
  52
  53 @Component({
  54   templateUrl: 'import.component.html'
  55 })
  56 export class ImportComponent implements OnInit, AfterViewInit, OnDestroy {
  57
  58     recordType: string;
  59     selectedQueue: ComboboxEntry; // freetext enabled
  60
  61     // used for applying a default queue ID value when we have
  62     // a load-time queue before the queue combobox entries exist.
  63     startQueueId: number;
  64
  65     bibTrashGroups: IdlObject[];
  66     selectedTrashGroups: number[];
  67
  68     activeQueueId: number;
  69     selectedBucket: number;
  70     selectedBibSource: number;
  71     selectedMatchSet: number;
  72     selectedHoldingsProfile: number;
  73     selectedMergeProfile: number;
  74     selectedFallThruMergeProfile: number;
  75     selectedFile: File;
  76
  77     defaultMatchSet: string;
  78
  79     importNonMatching: boolean;
  80     mergeOnExact: boolean;
  81     mergeOnSingleMatch: boolean;
  82     mergeOnBestMatch: boolean;
  83     minQualityRatio: number;
  84     autoOverlayAcqCopies: boolean;
  85
  86     // True after the first upload, then remains true.
  87     showProgress: boolean;
  88
  89     // Upload in progress.
  90     isUploading: boolean;
  91
  92     // True only after successful upload
  93     uploadComplete: boolean;
  94
  95     // Upload / processsing session key
  96     // Generated by the server
  97     sessionKey: string;
  98
  99     // Optional enqueue/import tracker session name.
 100     sessionName: string;
 101
 102     selectedTemplate: string;
 103     formTemplates: {[name: string]: any};
 104     newTemplateName: string;
 105
 106     @ViewChild('fileSelector') private fileSelector;
 107     @ViewChild('uploadProgress')
 108         private uploadProgress: ProgressInlineComponent;
 109     @ViewChild('enqueueProgress')
 110         private enqueueProgress: ProgressInlineComponent;
 111     @ViewChild('importProgress')
 112         private importProgress: ProgressInlineComponent;
 113
 114     // Need these refs so values can be applied via external stimuli
 115     @ViewChild('formTemplateSelector')
 116         private formTemplateSelector: ComboboxComponent;
 117     @ViewChild('recordTypeSelector')
 118         private recordTypeSelector: ComboboxComponent;
 119     @ViewChild('bibSourceSelector')
 120         private bibSourceSelector: ComboboxComponent;
 121     @ViewChild('matchSetSelector')
 122         private matchSetSelector: ComboboxComponent;
 123     @ViewChild('holdingsProfileSelector')
 124         private holdingsProfileSelector: ComboboxComponent;
 125     @ViewChild('mergeProfileSelector')
 126         private mergeProfileSelector: ComboboxComponent;
 127     @ViewChild('fallThruMergeProfileSelector')
 128         private fallThruMergeProfileSelector: ComboboxComponent;
 129
 130     @ViewChild('dupeQueueAlert')
 131         private dupeQueueAlert: AlertDialogComponent;
 132
 133     constructor(
 134         private http: HttpClient,
 135         private toast: ToastService,
 136         private evt: EventService,
 137         private net: NetService,
 138         private auth: AuthService,
 139         private org: OrgService,
 140         private store: ServerStoreService,
 141         private vandelay: VandelayService
 142     ) {
 143         this.applyDefaults();
 144     }
 145
 146     applyDefaults() {
 147         this.minQualityRatio = 0;
 148         this.selectedBibSource = 1; // default to system local
 149         this.recordType = 'bib';
 150         this.bibTrashGroups = [];
 151         this.formTemplates = {};
 152
 153         if (this.vandelay.importSelection) {
 154
 155             if (!this.vandelay.importSelection.queue) {
 156                 // Incomplete import selection, clear it.
 157                 this.vandelay.importSelection = null;
 158                 return;
 159             }
 160
 161             const queue = this.vandelay.importSelection.queue;
 162             this.recordType = queue.queue_type();
 163             this.selectedMatchSet = queue.match_set();
 164
 165             // This will be propagated to selectedQueue as a combobox
 166             // entry via the combobox
 167             this.startQueueId = queue.id();
 168
 169             if (this.recordType === 'bib') {
 170                 this.selectedBucket = queue.match_bucket();
 171                 this.selectedHoldingsProfile = queue.item_attr_def();
 172             }
 173         }
 174     }
 175
 176     ngOnInit() {}
 177
 178     ngAfterViewInit() {
 179         this.loadStartupData();
 180     }
 181
 182     ngOnDestroy() {
 183         // If we successfully completed the most recent
 184         // upload/import assume the importSelection can be cleared.
 185         if (this.uploadComplete) {
 186             this.clearSelection();
 187         }
 188     }
 189
 190     importSelection(): VandelayImportSelection {
 191         return this.vandelay.importSelection;
 192     }
 193
 194     loadStartupData(): Promise<any> {
 195         // Note displaying and manipulating a progress dialog inside
 196         // the AfterViewInit cycle leads to errors because the child
 197         // component is modifed after dirty checking.
 198
 199         const promises = [
 200             this.vandelay.getMergeProfiles(),
 201             this.vandelay.getAllQueues('bib'),
 202             this.vandelay.getAllQueues('authority'),
 203             this.vandelay.getMatchSets('bib'),
 204             this.vandelay.getMatchSets('authority'),
 205             this.vandelay.getBibBuckets(),
 206             this.vandelay.getBibSources(),
 207             this.vandelay.getItemImportDefs(),
 208             this.vandelay.getBibTrashGroups().then(
 209                 groups => this.bibTrashGroups = groups),
 210             this.org.settings(['vandelay.default_match_set']).then(
 211                 s => this.defaultMatchSet = s['vandelay.default_match_set']),
 212             this.loadTemplates()
 213         ];
 214
 215         return Promise.all(promises);
 216     }
 217
 218     loadTemplates() {
 219         this.store.getItem(TEMPLATE_SETTING_NAME).then(
 220             templates => {
 221                 this.formTemplates = templates || {};
 222
 223                 Object.keys(this.formTemplates).forEach(name => {
 224                     if (this.formTemplates[name].default) {
 225                         this.selectedTemplate = name;
 226                     }
 227                 });
 228             }
 229         );
 230     }
 231
 232     formatTemplateEntries(): ComboboxEntry[] {
 233         const entries = [];
 234
 235         Object.keys(this.formTemplates || {}).forEach(
 236             name => entries.push({id: name, label: name}));
 237
 238         return entries;
 239     }
 240
 241     // Format typeahead data sets
 242     formatEntries(etype: string): ComboboxEntry[] {
 243         const rtype = this.recordType;
 244         let list;
 245
 246         switch (etype) {
 247             case 'bibSources':
 248                 return (this.vandelay.bibSources || []).map(
 249                     s => { return {id: s.id(), label: s.source()}; });
 250
 251             case 'bibBuckets':
 252                 list = this.vandelay.bibBuckets;
 253                 break;
 254
 255             case 'activeQueues':
 256                 list = (this.vandelay.allQueues[rtype] || [])
 257                         .filter(q => q.complete() === 'f');
 258                 break;
 259
 260             case 'matchSets':
 261                 list = this.vandelay.matchSets[rtype];
 262                 break;
 263
 264             case 'importItemDefs':
 265                 list = this.vandelay.importItemAttrDefs;
 266                 break;
 267
 268             case 'mergeProfiles':
 269                 list = this.vandelay.mergeProfiles;
 270                 break;
 271         }
 272
 273         return (list || []).map(item => {
 274             return {id: item.id(), label: item.name()};
 275         });
 276     }
 277
 278     selectEntry($event: ComboboxEntry, etype: string) {
 279         const id = $event ? $event.id : null;
 280
 281         switch (etype) {
 282             case 'recordType':
 283                 this.recordType = id;
 284
 285             case 'bibSources':
 286                 this.selectedBibSource = id;
 287                 break;
 288
 289             case 'bibBuckets':
 290                 this.selectedBucket = id;
 291                 break;
 292
 293             case 'matchSets':
 294                 this.selectedMatchSet = id;
 295                 break;
 296
 297             case 'importItemDefs':
 298                 this.selectedHoldingsProfile = id;
 299                 break;
 300
 301             case 'mergeProfiles':
 302                 this.selectedMergeProfile = id;
 303                 break;
 304
 305             case 'FallThruMergeProfile':
 306                 this.selectedFallThruMergeProfile = id;
 307                 break;
 308         }
 309     }
 310
 311     fileSelected($event) {
 312        this.selectedFile = $event.target.files[0];
 313     }
 314
 315     // Required form data varies depending on context.
 316     hasNeededData(): boolean {
 317         if (this.vandelay.importSelection) {
 318             return this.importActionSelected();
 319         } else {
 320             return this.selectedQueue
 321                 && Boolean(this.recordType) && Boolean(this.selectedFile)
 322         }
 323     }
 324
 325     importActionSelected(): boolean {
 326         return this.importNonMatching
 327             || this.mergeOnExact
 328             || this.mergeOnSingleMatch
 329             || this.mergeOnBestMatch;
 330     }
 331
 332     // 1. create queue if necessary
 333     // 2. upload MARC file
 334     // 3. Enqueue MARC records
 335     // 4. Import records
 336     upload() {
 337         this.sessionKey = null;
 338         this.showProgress = true;
 339         this.isUploading = true;
 340         this.uploadComplete = false;
 341         this.resetProgressBars();
 342
 343         this.resolveQueue()
 344         .then(
 345             queueId => {
 346                 this.activeQueueId = queueId;
 347                 return this.uploadFile();
 348             },
 349             err => Promise.reject('queue create failed')
 350         ).then(
 351             ok => this.processSpool(),
 352             err => Promise.reject('process spool failed')
 353         ).then(
 354             ok => this.importRecords(),
 355             err => Promise.reject('import records failed')
 356         ).then(
 357             ok => {
 358                 this.isUploading = false;
 359                 this.uploadComplete = true;
 360             },
 361             err => {
 362                 console.log('file upload failed: ', err);
 363                 this.isUploading = false;
 364                 this.resetProgressBars();
 365
 366             }
 367         );
 368     }
 369
 370     resetProgressBars() {
 371         this.uploadProgress.update({value: 0, max: 1});
 372         this.enqueueProgress.update({value: 0, max: 1});
 373         this.importProgress.update({value: 0, max: 1});
 374     }
 375
 376     // Extract selected queue ID or create a new queue when requested.
 377     resolveQueue(): Promise<number> {
 378
 379         if (this.selectedQueue.freetext) {
 380             // Free text queue selector means create a new entry.
 381             // TODO: first check for name dupes
 382
 383             return this.vandelay.createQueue(
 384                 this.selectedQueue.label,
 385                 this.recordType,
 386                 this.selectedHoldingsProfile,
 387                 this.selectedMatchSet,
 388                 this.selectedBucket
 389             ).then(
 390                 id => id,
 391                 err => {
 392                     const evt = this.evt.parse(err);
 393                     if (evt) {
 394                         if (evt.textcode.match(/QUEUE_EXISTS/)) {
 395                             this.dupeQueueAlert.open();
 396                         } else {
 397                             alert(evt); // server error
 398                         }
 399                     }
 400
 401                     return Promise.reject('Queue Create Failed');
 402                 }
 403             );
 404         } else {
 405             return Promise.resolve(this.selectedQueue.id);
 406         }
 407     }
 408
 409     uploadFile(): Promise<any> {
 410
 411         if (this.vandelay.importSelection) {
 412             // Nothing to upload when processing pre-queued records.
 413             return Promise.resolve();
 414         }
 415
 416         const formData: FormData = new FormData();
 417
 418         formData.append('ses', this.auth.token());
 419         formData.append('marc_upload',
 420             this.selectedFile, this.selectedFile.name);
 421
 422         if (this.selectedBibSource) {
 423             formData.append('bib_source', ''+this.selectedBibSource);
 424         }
 425
 426         const req = new HttpRequest('POST', VANDELAY_UPLOAD_PATH, formData,
 427             {reportProgress: true, responseType: 'text'});
 428
 429         return this.http.request(req).pipe(tap(
 430             evt => {
 431                 if (evt.type === HttpEventType.UploadProgress) {
 432                     this.uploadProgress.update(
 433                         {value: evt.loaded, max: evt.total});
 434
 435                 } else if (evt instanceof HttpResponse) {
 436                     this.sessionKey = evt.body as string;
 437                     console.log(
 438                         'Vandelay file uploaded OK with key '+this.sessionKey);
 439                 }
 440             },
 441
 442             (err: HttpErrorResponse) => {
 443                 console.error(err);
 444                 this.toast.danger(err.error);
 445             }
 446         )).toPromise();
 447     }
 448
 449     processSpool():  Promise<any> {
 450
 451         if (this.vandelay.importSelection) {
 452             // Nothing to enqueue when processing pre-queued records
 453             return Promise.resolve();
 454         }
 455         var spoolType = this.recordType;
 456         if (this.recordType == 'authority') spoolType = 'auth'
 457
 458         const method = `open-ils.vandelay.${spoolType}.process_spool`;
 459
 460         return new Promise((resolve, reject) => {
 461             this.net.request(
 462                 'open-ils.vandelay', method,
 463                 this.auth.token(), this.sessionKey, this.activeQueueId,
 464                 null, null, this.selectedBibSource,
 465                 (this.sessionName || null), true
 466             ).subscribe(
 467                 tracker => {
 468                     const e = this.evt.parse(tracker);
 469                     if (e) { console.error(e); return reject(); }
 470
 471                     // Spooling is in progress, track the results.
 472                     this.vandelay.pollSessionTracker(tracker.id())
 473                     .subscribe(
 474                         trkr => {
 475                             this.enqueueProgress.update({
 476                                 // enqueue API only tracks actions performed
 477                                 max: null,
 478                                 value: trkr.actions_performed()
 479                             });
 480                         },
 481                         err => { console.log(err); reject(); },
 482                         () => {
 483                             this.enqueueProgress.update({max: 1, value: 1});
 484                             resolve();
 485                         }
 486                     );
 487                 }
 488             );
 489         });
 490     }
 491
 492     importRecords(): Promise<any> {
 493
 494         if (!this.importActionSelected()) {
 495             return Promise.resolve();
 496         }
 497
 498         const selection = this.vandelay.importSelection;
 499
 500         if (selection && !selection.importQueue) {
 501             return this.importRecordQueue(selection.recordIds);
 502         } else {
 503             return this.importRecordQueue();
 504         }
 505     }
 506
 507     importRecordQueue(recIds?: number[]): Promise<any> {
 508         const rtype = this.recordType === 'bib' ? 'bib' : 'auth';
 509
 510         let method = `open-ils.vandelay.${rtype}_queue.import`;
 511         const options: ImportOptions = this.compileImportOptions();
 512
 513         let target: number | number[] = this.activeQueueId;
 514         if (recIds && recIds.length) {
 515             method = `open-ils.vandelay.${rtype}_record.list.import`;
 516             target = recIds;
 517         }
 518
 519         return new Promise((resolve, reject) => {
 520             this.net.request('open-ils.vandelay',
 521                 method, this.auth.token(), target, options)
 522             .subscribe(
 523                 tracker => {
 524                     const e = this.evt.parse(tracker);
 525                     if (e) { console.error(e); return reject(); }
 526
 527                     // Spooling is in progress, track the results.
 528                     this.vandelay.pollSessionTracker(tracker.id())
 529                     .subscribe(
 530                         trkr => {
 531                             this.importProgress.update({
 532                                 max: trkr.total_actions(),
 533                                 value: trkr.actions_performed()
 534                             });
 535                         },
 536                         err => { console.log(err); reject(); },
 537                         () => {
 538                             this.importProgress.update({max: 1, value: 1});
 539                             resolve();
 540                         }
 541                     );
 542                 }
 543             );
 544         });
 545     }
 546
 547     compileImportOptions(): ImportOptions {
 548
 549         const options: ImportOptions = {
 550             session_key: this.sessionKey,
 551             import_no_match: this.importNonMatching,
 552             auto_overlay_exact: this.mergeOnExact,
 553             auto_overlay_best_match: this.mergeOnBestMatch,
 554             auto_overlay_1match: this.mergeOnSingleMatch,
 555             opp_acq_copy_overlay: this.autoOverlayAcqCopies,
 556             merge_profile: this.selectedMergeProfile,
 557             fall_through_merge_profile: this.selectedFallThruMergeProfile,
 558             strip_field_groups: this.selectedTrashGroups,
 559             match_quality_ratio: this.minQualityRatio,
 560             exit_early: true
 561         };
 562
 563         if (this.vandelay.importSelection) {
 564             options.overlay_map = this.vandelay.importSelection.overlayMap;
 565         }
 566
 567         return options;
 568     }
 569
 570     clearSelection() {
 571         this.vandelay.importSelection = null;
 572         this.startQueueId = null;
 573     }
 574
 575     openQueue() {
 576         console.log('opening queue ' + this.activeQueueId);
 577     }
 578
 579     saveTemplate() {
 580
 581         const template = {};
 582         TEMPLATE_ATTRS.forEach(key => template[key] = this[key]);
 583
 584         console.debug("Saving import profile", template);
 585
 586         this.formTemplates[this.selectedTemplate] = template;
 587         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 588     }
 589
 590     markTemplateDefault() {
 591
 592         Object.keys(this.formTemplates).forEach(
 593             name => delete this.formTemplates.default
 594         );
 595
 596         this.formTemplates[this.selectedTemplate].default = true;
 597
 598         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 599     }
 600
 601     templateSelectorChange(entry: ComboboxEntry) {
 602
 603         if (!entry) {
 604             this.selectedTemplate = '';
 605             return;
 606         }
 607
 608         this.selectedTemplate = entry.label; // label == name
 609
 610         if (entry.freetext) {
 611             // User is entering a new template name.
 612             // Nothing to apply.
 613             return;
 614         }
 615
 616         // User selected an existing template, apply it to the form.
 617
 618         const template = this.formTemplates[entry.id];
 619
 620         // Copy the template values into "this"
 621         TEMPLATE_ATTRS.forEach(key => this[key] = template[key]);
 622
 623         // Some values must be manually passed to the combobox'es
 624
 625         this.recordTypeSelector.applyEntryId(this.recordType);
 626         this.bibSourceSelector.applyEntryId(this.selectedBibSource);
 627         this.matchSetSelector.applyEntryId(this.selectedMatchSet);
 628         this.holdingsProfileSelector
 629             .applyEntryId(this.selectedHoldingsProfile);
 630         this.mergeProfileSelector.applyEntryId(this.selectedMergeProfile);
 631         this.fallThruMergeProfileSelector
 632             .applyEntryId(this.selectedFallThruMergeProfile);
 633     }
 634
 635     deleteTemplate() {
 636         delete this.formTemplates[this.selectedTemplate];
 637         this.formTemplateSelector.selected = null;
 638         return this.store.setItem(TEMPLATE_SETTING_NAME, this.formTemplates);
 639     }
 640 }
 641