fix: address PR review comments - tesseract extractor, dto, confidence, field mapper, drag-drop, migration

Co-authored-by: RobertgPatch <5817970+RobertgPatch@users.noreply.github.com>
Agent-Logs-Url: https://github.com/RobertgPatch/portfolio-management/sessions/7c542bd9-8d19-4eb5-bf68-6debfe42d8af

apps/api/src/app/k1-import/extractors/tesseract-extractor.ts

@@ -55,11 +55,14 @@ export class TesseractExtractor implements K1Extractor {
 
       // Fallback: try pdf-parse to at least get any embedded text
       try {
-        const { PDFParse } = await import('pdf-parse');
-        const parser = new PDFParse({ data: buffer });
-        const parsed = await parser.getText();
-        text = parsed.text;
-        pageCount = parsed.total;
+        const pdfParseModule = await import('pdf-parse');
+        const pdfParse = (pdfParseModule as any).default || pdfParseModule;
+        const parsed = await pdfParse(buffer);
+        text = parsed.text ?? '';
+        pageCount =
+          typeof parsed.numpages === 'number' && parsed.numpages > 0
+            ? parsed.numpages
+            : 1;
       } catch (parseError) {
         this.logger.error(
           `Both Tesseract and pdf-parse failed: ${parseError}`

apps/api/src/app/k1-import/k1-confidence.service.ts

@@ -105,7 +105,7 @@ export class K1ConfidenceService {
   public applyAutoReview(fields: K1ExtractedField[]): K1ExtractedField[] {
     return fields.map((field) => ({
       ...field,
-      isReviewed: field.confidenceLevel === 'HIGH'
+      isReviewed: field.isReviewed || field.confidenceLevel === 'HIGH'
     }));
   }
 }

apps/api/src/app/k1-import/k1-field-mapper.service.ts

@@ -50,7 +50,6 @@ export class K1FieldMapperService {
         mappedFields.push({
           ...field,
           label: mapping.label,
-          customLabel: mapping.isCustom ? mapping.label : field.customLabel,
           cellType: mapping.cellType
         } as any);
       } else {
@@ -121,7 +120,7 @@ export class K1FieldMapperService {
         missingFields.push({
           boxNumber: mapping.boxNumber,
           label: mapping.label,
-          customLabel: mapping.isCustom ? mapping.label : null,
+          customLabel: null,
           rawValue: '',
           numericValue: null,
           confidence: 1.0, // Empty fields have full confidence

apps/client/src/app/pages/k1-import/k1-import-page.component.ts

@@ -7,6 +7,7 @@ import {
   ChangeDetectorRef,
   Component,
   DestroyRef,
+  OnDestroy,
   OnInit
 } from '@angular/core';
 import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
@@ -44,7 +45,7 @@ import {
   styleUrls: ['./k1-import-page.scss'],
   templateUrl: './k1-import-page.html'
 })
-export class K1ImportPageComponent implements OnInit {
+export class K1ImportPageComponent implements OnDestroy, OnInit {
   public error: string | null = null;
   public extractionStatus: string | null = null;
   public historyColumns = ['createdAt', 'fileName', 'taxYear', 'status', 'kDocument', 'actions'];
@@ -75,6 +76,10 @@ export class K1ImportPageComponent implements OnInit {
     }
   }
 
+  public ngOnDestroy(): void {
+    this.stopPolling();
+  }
+
   public ngOnInit(): void {
     this.fetchPartnerships();
   }
@@ -125,29 +130,39 @@ export class K1ImportPageComponent implements OnInit {
   }
 
   public onFileSelected(event: Event): void {
-    const input = event.target as HTMLInputElement;
-    if (input.files && input.files.length > 0) {
-      const file = input.files[0];
-
-      // Client-side validation
-      if (file.type !== 'application/pdf') {
-        this.error = 'Please select a valid PDF file.';
-        this.selectedFile = null;
-        this.changeDetectorRef.markForCheck();
-        return;
-      }
+    let file: File | null = null;
 
-      if (file.size > 25 * 1024 * 1024) {
-        this.error = 'File exceeds 25 MB size limit.';
-        this.selectedFile = null;
-        this.changeDetectorRef.markForCheck();
-        return;
+    if (event instanceof DragEvent && event.dataTransfer?.files?.length) {
+      file = event.dataTransfer.files[0];
+    } else {
+      const input = event.target as HTMLInputElement;
+      if (input.files && input.files.length > 0) {
+        file = input.files[0];
       }
+    }
+
+    if (!file) {
+      return;
+    }
 
-      this.error = null;
-      this.selectedFile = file;
+    // Client-side validation
+    if (file.type !== 'application/pdf') {
+      this.error = 'Please select a valid PDF file.';
+      this.selectedFile = null;
       this.changeDetectorRef.markForCheck();
+      return;
+    }
+
+    if (file.size > 25 * 1024 * 1024) {
+      this.error = 'File exceeds 25 MB size limit.';
+      this.selectedFile = null;
+      this.changeDetectorRef.markForCheck();
+      return;
     }
+
+    this.error = null;
+    this.selectedFile = file;
+    this.changeDetectorRef.markForCheck();
   }
 
   public uploadK1(): void {

libs/common/src/lib/dtos/k1-import.dto.ts

@@ -51,6 +51,10 @@ export class K1ExtractedFieldDto {
   @IsBoolean()
   isReviewed: boolean;
 
+  @IsOptional()
+  @IsString()
+  cellType?: string;
+
   @IsOptional()
   @IsString()
   subtype?: string | null;

prisma/migrations/20260321000000_added_k1_import_tables/migration.sql

@@ -0,0 +1,93 @@
+-- CreateEnum
+CREATE TYPE "K1ImportStatus" AS ENUM ('PROCESSING', 'EXTRACTED', 'VERIFIED', 'CONFIRMED', 'CANCELLED', 'FAILED');
+
+-- CreateTable
+CREATE TABLE "K1ImportSession" (
+    "id" TEXT NOT NULL,
+    "partnershipId" TEXT NOT NULL,
+    "userId" TEXT NOT NULL,
+    "status" "K1ImportStatus" NOT NULL DEFAULT 'PROCESSING',
+    "taxYear" INTEGER NOT NULL,
+    "fileName" TEXT NOT NULL,
+    "fileSize" INTEGER NOT NULL,
+    "extractionMethod" TEXT NOT NULL,
+    "rawExtraction" JSONB,
+    "verifiedData" JSONB,
+    "documentId" TEXT,
+    "kDocumentId" TEXT,
+    "errorMessage" TEXT,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "K1ImportSession_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "CellMapping" (
+    "id" TEXT NOT NULL,
+    "partnershipId" TEXT,
+    "boxNumber" TEXT NOT NULL,
+    "label" TEXT NOT NULL,
+    "description" TEXT,
+    "cellType" TEXT NOT NULL DEFAULT 'number',
+    "isCustom" BOOLEAN NOT NULL DEFAULT false,
+    "isIgnored" BOOLEAN NOT NULL DEFAULT false,
+    "sortOrder" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "CellMapping_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "CellAggregationRule" (
+    "id" TEXT NOT NULL,
+    "partnershipId" TEXT,
+    "name" TEXT NOT NULL,
+    "operation" TEXT NOT NULL DEFAULT 'SUM',
+    "sourceCells" JSONB NOT NULL,
+    "sortOrder" INTEGER NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "CellAggregationRule_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "K1ImportSession_partnershipId_taxYear_idx" ON "K1ImportSession"("partnershipId", "taxYear");
+
+-- CreateIndex
+CREATE INDEX "K1ImportSession_userId_idx" ON "K1ImportSession"("userId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "K1ImportSession_kDocumentId_key" ON "K1ImportSession"("kDocumentId");
+
+-- CreateIndex
+CREATE INDEX "CellMapping_partnershipId_idx" ON "CellMapping"("partnershipId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "CellMapping_partnershipId_boxNumber_key" ON "CellMapping"("partnershipId", "boxNumber");
+
+-- CreateIndex
+CREATE INDEX "CellAggregationRule_partnershipId_idx" ON "CellAggregationRule"("partnershipId");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "CellAggregationRule_partnershipId_name_key" ON "CellAggregationRule"("partnershipId", "name");
+
+-- AddForeignKey
+ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "Document"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_kDocumentId_fkey" FOREIGN KEY ("kDocumentId") REFERENCES "KDocument"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "CellMapping" ADD CONSTRAINT "CellMapping_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "CellAggregationRule" ADD CONSTRAINT "CellAggregationRule_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE;

tmp-check-users.mjs

@@ -1,21 +0,0 @@
-import { PrismaClient } from '@prisma/client';
-const p = new PrismaClient();
-
-// Delete all data in dependency order
-await p.access.deleteMany();
-await p.order.deleteMany();
-await p.accountBalance.deleteMany();
-await p.account.deleteMany();
-await p.symbolProfile.deleteMany();
-await p.marketData.deleteMany();
-await p.settings.deleteMany();
-await p.subscription.deleteMany();
-await p.authDevice.deleteMany();
-await p.analytics.deleteMany();
-await p.user.deleteMany();
-
-console.log('All users deleted.');
-
-const users = await p.user.findMany({ select: { id: true, role: true } });
-console.log('USERS after delete:', JSON.stringify(users));
-await p.$disconnect();