From 318ff997041b88deece60ca1431f99339cec0d91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 08:23:56 +0000 Subject: [PATCH 1/2] Initial plan From 558ccebcd2baf6bfc48bd6438f3532134091582d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 21 Mar 2026 08:28:24 +0000 Subject: [PATCH 2/2] fix: address PR review comments - tesseract extractor, dto, confidence, field mapper, drag-drop, migration Co-authored-by: RobertgPatch <5817970+RobertgPatch@users.noreply.github.com> Agent-Logs-Url: https://github.com/RobertgPatch/portfolio-management/sessions/7c542bd9-8d19-4eb5-bf68-6debfe42d8af --- .../extractors/tesseract-extractor.ts | 13 ++- .../app/k1-import/k1-confidence.service.ts | 2 +- .../app/k1-import/k1-field-mapper.service.ts | 3 +- .../k1-import/k1-import-page.component.ts | 53 +++++++---- libs/common/src/lib/dtos/k1-import.dto.ts | 4 + .../migration.sql | 93 +++++++++++++++++++ tmp-check-users.mjs | 21 ----- 7 files changed, 141 insertions(+), 48 deletions(-) create mode 100644 prisma/migrations/20260321000000_added_k1_import_tables/migration.sql delete mode 100644 tmp-check-users.mjs diff --git a/apps/api/src/app/k1-import/extractors/tesseract-extractor.ts b/apps/api/src/app/k1-import/extractors/tesseract-extractor.ts index b2f26e637..a8b255be4 100644 --- a/apps/api/src/app/k1-import/extractors/tesseract-extractor.ts +++ b/apps/api/src/app/k1-import/extractors/tesseract-extractor.ts @@ -55,11 +55,14 @@ export class TesseractExtractor implements K1Extractor { // Fallback: try pdf-parse to at least get any embedded text try { - const { PDFParse } = await import('pdf-parse'); - const parser = new PDFParse({ data: buffer }); - const parsed = await parser.getText(); - text = parsed.text; - pageCount = parsed.total; + const pdfParseModule = await import('pdf-parse'); + const pdfParse = (pdfParseModule as any).default || pdfParseModule; + const parsed = await pdfParse(buffer); + text = parsed.text ?? ''; + pageCount = + typeof parsed.numpages === 'number' && parsed.numpages > 0 + ? parsed.numpages + : 1; } catch (parseError) { this.logger.error( `Both Tesseract and pdf-parse failed: ${parseError}` diff --git a/apps/api/src/app/k1-import/k1-confidence.service.ts b/apps/api/src/app/k1-import/k1-confidence.service.ts index db7631b67..5b0655e99 100644 --- a/apps/api/src/app/k1-import/k1-confidence.service.ts +++ b/apps/api/src/app/k1-import/k1-confidence.service.ts @@ -105,7 +105,7 @@ export class K1ConfidenceService { public applyAutoReview(fields: K1ExtractedField[]): K1ExtractedField[] { return fields.map((field) => ({ ...field, - isReviewed: field.confidenceLevel === 'HIGH' + isReviewed: field.isReviewed || field.confidenceLevel === 'HIGH' })); } } diff --git a/apps/api/src/app/k1-import/k1-field-mapper.service.ts b/apps/api/src/app/k1-import/k1-field-mapper.service.ts index 832ba3387..bed91cac3 100644 --- a/apps/api/src/app/k1-import/k1-field-mapper.service.ts +++ b/apps/api/src/app/k1-import/k1-field-mapper.service.ts @@ -50,7 +50,6 @@ export class K1FieldMapperService { mappedFields.push({ ...field, label: mapping.label, - customLabel: mapping.isCustom ? mapping.label : field.customLabel, cellType: mapping.cellType } as any); } else { @@ -121,7 +120,7 @@ export class K1FieldMapperService { missingFields.push({ boxNumber: mapping.boxNumber, label: mapping.label, - customLabel: mapping.isCustom ? mapping.label : null, + customLabel: null, rawValue: '', numericValue: null, confidence: 1.0, // Empty fields have full confidence diff --git a/apps/client/src/app/pages/k1-import/k1-import-page.component.ts b/apps/client/src/app/pages/k1-import/k1-import-page.component.ts index 5cb41ee39..055c054a4 100644 --- a/apps/client/src/app/pages/k1-import/k1-import-page.component.ts +++ b/apps/client/src/app/pages/k1-import/k1-import-page.component.ts @@ -7,6 +7,7 @@ import { ChangeDetectorRef, Component, DestroyRef, + OnDestroy, OnInit } from '@angular/core'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -44,7 +45,7 @@ import { styleUrls: ['./k1-import-page.scss'], templateUrl: './k1-import-page.html' }) -export class K1ImportPageComponent implements OnInit { +export class K1ImportPageComponent implements OnDestroy, OnInit { public error: string | null = null; public extractionStatus: string | null = null; public historyColumns = ['createdAt', 'fileName', 'taxYear', 'status', 'kDocument', 'actions']; @@ -75,6 +76,10 @@ export class K1ImportPageComponent implements OnInit { } } + public ngOnDestroy(): void { + this.stopPolling(); + } + public ngOnInit(): void { this.fetchPartnerships(); } @@ -125,29 +130,39 @@ export class K1ImportPageComponent implements OnInit { } public onFileSelected(event: Event): void { - const input = event.target as HTMLInputElement; - if (input.files && input.files.length > 0) { - const file = input.files[0]; - - // Client-side validation - if (file.type !== 'application/pdf') { - this.error = 'Please select a valid PDF file.'; - this.selectedFile = null; - this.changeDetectorRef.markForCheck(); - return; - } + let file: File | null = null; - if (file.size > 25 * 1024 * 1024) { - this.error = 'File exceeds 25 MB size limit.'; - this.selectedFile = null; - this.changeDetectorRef.markForCheck(); - return; + if (event instanceof DragEvent && event.dataTransfer?.files?.length) { + file = event.dataTransfer.files[0]; + } else { + const input = event.target as HTMLInputElement; + if (input.files && input.files.length > 0) { + file = input.files[0]; } + } + + if (!file) { + return; + } - this.error = null; - this.selectedFile = file; + // Client-side validation + if (file.type !== 'application/pdf') { + this.error = 'Please select a valid PDF file.'; + this.selectedFile = null; this.changeDetectorRef.markForCheck(); + return; + } + + if (file.size > 25 * 1024 * 1024) { + this.error = 'File exceeds 25 MB size limit.'; + this.selectedFile = null; + this.changeDetectorRef.markForCheck(); + return; } + + this.error = null; + this.selectedFile = file; + this.changeDetectorRef.markForCheck(); } public uploadK1(): void { diff --git a/libs/common/src/lib/dtos/k1-import.dto.ts b/libs/common/src/lib/dtos/k1-import.dto.ts index 80da7bfba..ea7aedab4 100644 --- a/libs/common/src/lib/dtos/k1-import.dto.ts +++ b/libs/common/src/lib/dtos/k1-import.dto.ts @@ -51,6 +51,10 @@ export class K1ExtractedFieldDto { @IsBoolean() isReviewed: boolean; + @IsOptional() + @IsString() + cellType?: string; + @IsOptional() @IsString() subtype?: string | null; diff --git a/prisma/migrations/20260321000000_added_k1_import_tables/migration.sql b/prisma/migrations/20260321000000_added_k1_import_tables/migration.sql new file mode 100644 index 000000000..df33ae130 --- /dev/null +++ b/prisma/migrations/20260321000000_added_k1_import_tables/migration.sql @@ -0,0 +1,93 @@ +-- CreateEnum +CREATE TYPE "K1ImportStatus" AS ENUM ('PROCESSING', 'EXTRACTED', 'VERIFIED', 'CONFIRMED', 'CANCELLED', 'FAILED'); + +-- CreateTable +CREATE TABLE "K1ImportSession" ( + "id" TEXT NOT NULL, + "partnershipId" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "status" "K1ImportStatus" NOT NULL DEFAULT 'PROCESSING', + "taxYear" INTEGER NOT NULL, + "fileName" TEXT NOT NULL, + "fileSize" INTEGER NOT NULL, + "extractionMethod" TEXT NOT NULL, + "rawExtraction" JSONB, + "verifiedData" JSONB, + "documentId" TEXT, + "kDocumentId" TEXT, + "errorMessage" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "K1ImportSession_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "CellMapping" ( + "id" TEXT NOT NULL, + "partnershipId" TEXT, + "boxNumber" TEXT NOT NULL, + "label" TEXT NOT NULL, + "description" TEXT, + "cellType" TEXT NOT NULL DEFAULT 'number', + "isCustom" BOOLEAN NOT NULL DEFAULT false, + "isIgnored" BOOLEAN NOT NULL DEFAULT false, + "sortOrder" INTEGER NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "CellMapping_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "CellAggregationRule" ( + "id" TEXT NOT NULL, + "partnershipId" TEXT, + "name" TEXT NOT NULL, + "operation" TEXT NOT NULL DEFAULT 'SUM', + "sourceCells" JSONB NOT NULL, + "sortOrder" INTEGER NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "CellAggregationRule_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "K1ImportSession_partnershipId_taxYear_idx" ON "K1ImportSession"("partnershipId", "taxYear"); + +-- CreateIndex +CREATE INDEX "K1ImportSession_userId_idx" ON "K1ImportSession"("userId"); + +-- CreateIndex +CREATE UNIQUE INDEX "K1ImportSession_kDocumentId_key" ON "K1ImportSession"("kDocumentId"); + +-- CreateIndex +CREATE INDEX "CellMapping_partnershipId_idx" ON "CellMapping"("partnershipId"); + +-- CreateIndex +CREATE UNIQUE INDEX "CellMapping_partnershipId_boxNumber_key" ON "CellMapping"("partnershipId", "boxNumber"); + +-- CreateIndex +CREATE INDEX "CellAggregationRule_partnershipId_idx" ON "CellAggregationRule"("partnershipId"); + +-- CreateIndex +CREATE UNIQUE INDEX "CellAggregationRule_partnershipId_name_key" ON "CellAggregationRule"("partnershipId", "name"); + +-- AddForeignKey +ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "Document"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "K1ImportSession" ADD CONSTRAINT "K1ImportSession_kDocumentId_fkey" FOREIGN KEY ("kDocumentId") REFERENCES "KDocument"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "CellMapping" ADD CONSTRAINT "CellMapping_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "CellAggregationRule" ADD CONSTRAINT "CellAggregationRule_partnershipId_fkey" FOREIGN KEY ("partnershipId") REFERENCES "Partnership"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/tmp-check-users.mjs b/tmp-check-users.mjs deleted file mode 100644 index be250484a..000000000 --- a/tmp-check-users.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import { PrismaClient } from '@prisma/client'; -const p = new PrismaClient(); - -// Delete all data in dependency order -await p.access.deleteMany(); -await p.order.deleteMany(); -await p.accountBalance.deleteMany(); -await p.account.deleteMany(); -await p.symbolProfile.deleteMany(); -await p.marketData.deleteMany(); -await p.settings.deleteMany(); -await p.subscription.deleteMany(); -await p.authDevice.deleteMany(); -await p.analytics.deleteMany(); -await p.user.deleteMany(); - -console.log('All users deleted.'); - -const users = await p.user.findMany({ select: { id: true, role: true } }); -console.log('USERS after delete:', JSON.stringify(users)); -await p.$disconnect();