diff --git a/apps/api/src/services/configuration/configuration.service.ts b/apps/api/src/services/configuration/configuration.service.ts index ad8e84a99..16b17d806 100644 --- a/apps/api/src/services/configuration/configuration.service.ts +++ b/apps/api/src/services/configuration/configuration.service.ts @@ -23,6 +23,8 @@ export class ConfigurationService { this.environmentConfiguration = cleanEnv(process.env, { ACCESS_TOKEN_SALT: str(), API_KEY_ALPHA_VANTAGE: str({ default: '' }), + AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT: str({ default: '' }), + AZURE_DOCUMENT_INTELLIGENCE_KEY: str({ default: '' }), API_KEY_BETTER_UPTIME: str({ default: '' }), API_KEY_COINGECKO_DEMO: str({ default: '' }), API_KEY_COINGECKO_PRO: str({ default: '' }), diff --git a/apps/api/src/services/interfaces/environment.interface.ts b/apps/api/src/services/interfaces/environment.interface.ts index 9664ae144..728031cdb 100644 --- a/apps/api/src/services/interfaces/environment.interface.ts +++ b/apps/api/src/services/interfaces/environment.interface.ts @@ -3,6 +3,8 @@ import { CleanedEnvAccessors } from 'envalid'; export interface Environment extends CleanedEnvAccessors { ACCESS_TOKEN_SALT: string; API_KEY_ALPHA_VANTAGE: string; + AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT: string; + AZURE_DOCUMENT_INTELLIGENCE_KEY: string; API_KEY_BETTER_UPTIME: string; API_KEY_COINGECKO_DEMO: string; API_KEY_COINGECKO_PRO: string; diff --git a/libs/common/src/lib/dtos/index.ts b/libs/common/src/lib/dtos/index.ts index 75dc88a96..0176d7949 100644 --- a/libs/common/src/lib/dtos/index.ts +++ b/libs/common/src/lib/dtos/index.ts @@ -14,6 +14,13 @@ import { CreateTagDto } from './create-tag.dto'; import { CreateWatchlistItemDto } from './create-watchlist-item.dto'; import { DeleteOwnUserDto } from './delete-own-user.dto'; import { CreateKDocumentDto, UpdateKDocumentDto } from './k-document.dto'; +import { + ConfirmK1ImportDto, + CreateK1ImportDto, + K1ExtractedFieldDto, + K1UnmappedItemDto, + VerifyK1ImportDto +} from './k1-import.dto'; import { CreatePartnershipAssetDto, CreatePartnershipDto, @@ -37,6 +44,7 @@ import { UpdateUserSettingDto } from './update-user-setting.dto'; export { AuthDeviceDto, + ConfirmK1ImportDto, CreateAccessDto, CreateAccountBalanceDto, CreateAccountDto, @@ -45,6 +53,7 @@ export { CreateAssetProfileWithMarketDataDto, CreateDistributionDto, CreateEntityDto, + CreateK1ImportDto, CreateKDocumentDto, CreateOrderDto, CreateOwnershipDto, @@ -56,6 +65,8 @@ export { CreateTagDto, CreateWatchlistItemDto, DeleteOwnUserDto, + K1ExtractedFieldDto, + K1UnmappedItemDto, TransferBalanceDto, UpdateAccessDto, UpdateAccountDto, @@ -70,5 +81,6 @@ export { UpdatePlatformDto, UpdatePropertyDto, UpdateTagDto, - UpdateUserSettingDto + UpdateUserSettingDto, + VerifyK1ImportDto }; diff --git a/libs/common/src/lib/dtos/k1-import.dto.ts b/libs/common/src/lib/dtos/k1-import.dto.ts new file mode 100644 index 000000000..fe5cc1683 --- /dev/null +++ b/libs/common/src/lib/dtos/k1-import.dto.ts @@ -0,0 +1,105 @@ +import { K1ImportStatus, KDocumentStatus } from '@prisma/client'; +import { + IsArray, + IsBoolean, + IsEnum, + IsInt, + IsNumber, + IsObject, + IsOptional, + IsString, + Min, + ValidateNested +} from 'class-validator'; +import { Type } from 'class-transformer'; + +export class CreateK1ImportDto { + @IsString() + partnershipId: string; + + @IsInt() + @Min(1900) + taxYear: number; +} + +export class K1ExtractedFieldDto { + @IsString() + boxNumber: string; + + @IsString() + label: string; + + @IsOptional() + @IsString() + customLabel?: string; + + @IsString() + rawValue: string; + + @IsOptional() + @IsNumber() + numericValue?: number; + + @IsNumber() + confidence: number; + + @IsString() + confidenceLevel: 'HIGH' | 'MEDIUM' | 'LOW'; + + @IsBoolean() + isUserEdited: boolean; + + @IsBoolean() + isReviewed: boolean; +} + +export class K1UnmappedItemDto { + @IsString() + rawLabel: string; + + @IsString() + rawValue: string; + + @IsOptional() + @IsNumber() + numericValue?: number; + + @IsNumber() + confidence: number; + + @IsInt() + pageNumber: number; + + @IsString() + resolution: 'assigned' | 'discarded'; + + @IsOptional() + @IsString() + assignedBoxNumber?: string; +} + +export class VerifyK1ImportDto { + @IsInt() + @Min(1900) + taxYear: number; + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => K1ExtractedFieldDto) + fields: K1ExtractedFieldDto[]; + + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => K1UnmappedItemDto) + unmappedItems?: K1UnmappedItemDto[]; +} + +export class ConfirmK1ImportDto { + @IsEnum(KDocumentStatus) + filingStatus: KDocumentStatus; + + @IsOptional() + @IsString() + existingKDocumentAction?: 'UPDATE' | 'CREATE_NEW'; +} diff --git a/libs/common/src/lib/interfaces/index.ts b/libs/common/src/lib/interfaces/index.ts index 29fac0925..1e2528f60 100644 --- a/libs/common/src/lib/interfaces/index.ts +++ b/libs/common/src/lib/interfaces/index.ts @@ -54,6 +54,14 @@ import type { IKDocumentAllocation, K1Data } from './k-document.interface'; +import type { + K1AggregationResult, + K1ConfirmationRequest, + K1ExtractionResult, + K1ExtractedField, + K1ImportSessionSummary, + K1UnmappedItem +} from './k1-import.interface'; import type { LineChartItem } from './line-chart-item.interface'; import type { LookupItem } from './lookup-item.interface'; import type { MarketData } from './market-data.interface'; @@ -192,6 +200,12 @@ export { IKDocument, IKDocumentAllocation, IOwnership, + K1AggregationResult, + K1ConfirmationRequest, + K1ExtractionResult, + K1ExtractedField, + K1ImportSessionSummary, + K1UnmappedItem, IPartnership, IPartnershipAsset, IPartnershipDetail, diff --git a/libs/common/src/lib/interfaces/k1-import.interface.ts b/libs/common/src/lib/interfaces/k1-import.interface.ts new file mode 100644 index 000000000..9f8b265a5 --- /dev/null +++ b/libs/common/src/lib/interfaces/k1-import.interface.ts @@ -0,0 +1,116 @@ +export interface K1ExtractionResult { + /** Extracted metadata from the K-1 header */ + metadata: { + partnershipName: string | null; + partnershipEin: string | null; + partnerName: string | null; + partnerEin: string | null; + taxYear: number | null; + isAmended: boolean; + isFinal: boolean; + }; + + /** Extracted box values — mapped to known cells */ + fields: K1ExtractedField[]; + + /** Extracted values that didn't match any configured cell mapping */ + unmappedItems: K1UnmappedItem[]; + + /** Overall extraction confidence (0.0–1.0) */ + overallConfidence: number; + + /** Extraction method used */ + method: 'pdf-parse' | 'azure' | 'tesseract'; + + /** Number of pages processed */ + pagesProcessed: number; +} + +export interface K1ExtractedField { + /** Box identifier (e.g., "1", "6a", "19a") */ + boxNumber: string; + + /** Display label from cell mapping */ + label: string; + + /** Custom label override by user (null if not overridden) */ + customLabel: string | null; + + /** Extracted raw text value */ + rawValue: string; + + /** Parsed numeric value (null if unparseable) */ + numericValue: number | null; + + /** Confidence score (0.0–1.0) */ + confidence: number; + + /** Confidence level for display */ + confidenceLevel: 'HIGH' | 'MEDIUM' | 'LOW'; + + /** Whether user has manually edited this value */ + isUserEdited: boolean; + + /** Whether user has explicitly reviewed this field (required for medium/low confidence) */ + isReviewed: boolean; +} + +export interface K1UnmappedItem { + /** Raw text label extracted from the PDF */ + rawLabel: string; + + /** Raw text value extracted */ + rawValue: string; + + /** Parsed numeric value (null if unparseable) */ + numericValue: number | null; + + /** Confidence score (0.0–1.0) */ + confidence: number; + + /** Page number where this was extracted */ + pageNumber: number; + + /** User action: 'assigned' (to a cell), 'discarded', or null (pending) */ + resolution: 'assigned' | 'discarded' | null; + + /** If assigned, the box number it was assigned to */ + assignedBoxNumber: string | null; +} + +export interface K1ConfirmationRequest { + /** Import session ID */ + importSessionId: string; + + /** Tax year (may have been overridden by user) */ + taxYear: number; + + /** Filing status for the new KDocument */ + filingStatus: 'DRAFT' | 'ESTIMATED' | 'FINAL'; + + /** Verified fields with any user edits applied */ + fields: K1ExtractedField[]; + + /** Whether to update an existing KDocument (null = create new) */ + existingKDocumentAction: 'UPDATE' | 'CREATE_NEW' | null; +} + +export interface K1ImportSessionSummary { + id: string; + partnershipId: string; + status: string; + taxYear: number; + fileName: string; + extractionMethod: string; + kDocumentId: string | null; + createdAt: string; +} + +export interface K1AggregationResult { + ruleId: string; + name: string; + operation: string; + sourceCells: string[]; + computedValue: number; + breakdown: Record; +} diff --git a/package-lock.json b/package-lock.json index e77522a6f..55718878f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "@angular/platform-browser-dynamic": "21.1.1", "@angular/router": "21.1.1", "@angular/service-worker": "21.1.1", + "@azure/ai-form-recognizer": "^5.1.0", "@bull-board/api": "6.20.3", "@bull-board/express": "6.20.3", "@bull-board/nestjs": "6.20.3", @@ -87,11 +88,13 @@ "passport-headerapikey": "1.2.2", "passport-jwt": "4.0.1", "passport-openidconnect": "0.1.2", + "pdf-parse": "^2.4.5", "reflect-metadata": "0.2.2", "rxjs": "7.8.1", "stripe": "20.3.0", "svgmap": "2.19.2", "tablemark": "4.1.0", + "tesseract.js": "^7.0.0", "twitter-api-v2": "1.29.0", "yahoo-finance2": "3.13.2", "zone.js": "0.16.0" @@ -137,6 +140,7 @@ "@types/papaparse": "5.3.7", "@types/passport-google-oauth20": "2.0.17", "@types/passport-openidconnect": "0.1.3", + "@types/pdf-parse": "^1.1.5", "@typescript-eslint/eslint-plugin": "8.43.0", "@typescript-eslint/parser": "8.43.0", "eslint": "9.35.0", @@ -1698,6 +1702,154 @@ "license": "MIT", "peer": true }, + "node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/ai-form-recognizer": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@azure/ai-form-recognizer/-/ai-form-recognizer-5.1.0.tgz", + "integrity": "sha512-XH6Nyj8+F/O3fH9RhHRUSSFkYMTJrDbw8F8M2mXm8jDkE06KQL0EDD9MTN9uLf+pZiYUWsEOQD9bPnLEtoh+lQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.0", + "@azure/core-tracing": "^1.2.0", + "@azure/logger": "^1.1.4", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", + "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", + "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", + "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.28.6.tgz", @@ -7249,6 +7401,190 @@ "win32" ] }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/nice": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@napi-rs/nice/-/nice-1.1.1.tgz", @@ -13416,6 +13752,16 @@ "@types/passport": "*" } }, + "node_modules/@types/pdf-parse": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/pdf-parse/-/pdf-parse-1.1.5.tgz", + "integrity": "sha512-kBfrSXsloMnUJOKi25s3+hRmkycHfLK6A09eRGqF/N8BkQoPUmaCr+q8Cli5FnfohEz/rsv82zAiPz/LXtOGhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/qs": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", @@ -13823,6 +14169,20 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.4.tgz", + "integrity": "sha512-CI0NhTrz4EBaa0U+HaaUZrJhPoso8sG7ZFya8uQoBA57fjzrjRSv87ekCjLZOFExN+gXE/z0xuN2QfH4H2HrLQ==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -15609,6 +15969,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", @@ -22017,7 +22383,6 @@ "version": "7.0.2", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^7.1.0", @@ -22182,6 +22547,12 @@ "postcss": "^8.1.0" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/identity-obj-proxy": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/identity-obj-proxy/-/identity-obj-proxy-3.0.0.tgz", @@ -22971,7 +23342,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", - "dev": true, "license": "MIT" }, "node_modules/is-weakmap": { @@ -27361,6 +27731,15 @@ "integrity": "sha512-vCseG/EQ6/RcvxhUcGJiHViOgrtz4x0XbZepXvKik66TMGkvbmjeJrKFyBEx6daG5rNyyd14zYXhz0hZVwQFOw==", "license": "MIT" }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/opener": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", @@ -28179,6 +28558,38 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/perfect-debounce": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/perfect-debounce/-/perfect-debounce-1.0.0.tgz", @@ -29674,6 +30085,12 @@ "node": ">=4" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/regex-parser": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.3.1.tgz", @@ -32749,6 +33166,30 @@ "devOptional": true, "license": "MIT" }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -34279,6 +34720,12 @@ "makeerror": "1.0.12" } }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, "node_modules/watchpack": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.0.tgz", @@ -35723,6 +36170,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/package.json b/package.json index e8b38049b..3bd1a0cfc 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "@angular/platform-browser-dynamic": "21.1.1", "@angular/router": "21.1.1", "@angular/service-worker": "21.1.1", + "@azure/ai-form-recognizer": "^5.1.0", "@bull-board/api": "6.20.3", "@bull-board/express": "6.20.3", "@bull-board/nestjs": "6.20.3", @@ -132,11 +133,13 @@ "passport-headerapikey": "1.2.2", "passport-jwt": "4.0.1", "passport-openidconnect": "0.1.2", + "pdf-parse": "^2.4.5", "reflect-metadata": "0.2.2", "rxjs": "7.8.1", "stripe": "20.3.0", "svgmap": "2.19.2", "tablemark": "4.1.0", + "tesseract.js": "^7.0.0", "twitter-api-v2": "1.29.0", "yahoo-finance2": "3.13.2", "zone.js": "0.16.0" @@ -182,6 +185,7 @@ "@types/papaparse": "5.3.7", "@types/passport-google-oauth20": "2.0.17", "@types/passport-openidconnect": "0.1.3", + "@types/pdf-parse": "^1.1.5", "@typescript-eslint/eslint-plugin": "8.43.0", "@typescript-eslint/parser": "8.43.0", "eslint": "9.35.0", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a7351febd..47289671e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -282,6 +282,7 @@ model User { updatedAt DateTime @updatedAt watchlist SymbolProfile[] @relation("UserWatchlist") SymbolProfile SymbolProfile[] + k1ImportSessions K1ImportSession[] @@index([accessToken]) @@index([createdAt]) @@ -468,8 +469,11 @@ model Partnership { assets PartnershipAsset[] valuations PartnershipValuation[] distributions Distribution[] - kDocuments KDocument[] - documents Document[] + kDocuments KDocument[] + documents Document[] + importSessions K1ImportSession[] + cellMappings CellMapping[] + aggregationRules CellAggregationRule[] @@index([name]) @@index([type]) @@ -549,6 +553,8 @@ model KDocument { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + importSession K1ImportSession? + @@unique([partnershipId, type, taxYear]) @@index([partnershipId]) @@index([taxYear]) @@ -620,8 +626,74 @@ model Document { taxYear Int? createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - kDocuments KDocument[] + kDocuments KDocument[] + k1ImportSessions K1ImportSession[] @@index([entityId]) @@index([partnershipId]) } + +enum K1ImportStatus { + PROCESSING + EXTRACTED + VERIFIED + CONFIRMED + CANCELLED + FAILED +} + +model K1ImportSession { + id String @id @default(uuid()) + partnershipId String + partnership Partnership @relation(fields: [partnershipId], onDelete: Cascade, references: [id]) + userId String + user User @relation(fields: [userId], onDelete: Cascade, references: [id]) + status K1ImportStatus @default(PROCESSING) + taxYear Int + fileName String + fileSize Int + extractionMethod String + rawExtraction Json? + verifiedData Json? + documentId String? + document Document? @relation(fields: [documentId], references: [id]) + kDocumentId String? @unique + kDocument KDocument? @relation(fields: [kDocumentId], references: [id]) + errorMessage String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([partnershipId, taxYear]) + @@index([userId]) +} + +model CellMapping { + id String @id @default(uuid()) + partnershipId String? + partnership Partnership? @relation(fields: [partnershipId], onDelete: Cascade, references: [id]) + boxNumber String + label String + description String? + isCustom Boolean @default(false) + sortOrder Int + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([partnershipId, boxNumber]) + @@index([partnershipId]) +} + +model CellAggregationRule { + id String @id @default(uuid()) + partnershipId String? + partnership Partnership? @relation(fields: [partnershipId], onDelete: Cascade, references: [id]) + name String + operation String @default("SUM") + sourceCells Json + sortOrder Int + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([partnershipId, name]) + @@index([partnershipId]) +} diff --git a/specs/004-k1-scan-import/tasks.md b/specs/004-k1-scan-import/tasks.md index fec74b46c..e56c40824 100644 --- a/specs/004-k1-scan-import/tasks.md +++ b/specs/004-k1-scan-import/tasks.md @@ -27,12 +27,12 @@ **Purpose**: Install dependencies, define database schema, create shared types and configuration -- [ ] T001 Install npm dependencies: pdf-parse, @azure/ai-form-recognizer, tesseract.js, @types/pdf-parse in package.json -- [ ] T002 [P] Add K1ImportStatus enum (PROCESSING, EXTRACTED, VERIFIED, CONFIRMED, CANCELLED, FAILED), K1ImportSession model, CellMapping model, and CellAggregationRule model to prisma/schema.prisma -- [ ] T003 [P] Create shared K-1 TypeScript interfaces (K1ExtractionResult, K1ExtractedField, K1UnmappedItem, K1ConfirmationRequest) in libs/common/src/lib/interfaces/k1-import.interface.ts -- [ ] T004 [P] Register AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT and AZURE_DOCUMENT_INTELLIGENCE_KEY environment variables in apps/api/src/app/configuration/configuration.service.ts -- [ ] T005 Run Prisma migration to create K-1 import tables and K1ImportStatus enum -- [ ] T006 [P] Create shared DTOs (CreateK1ImportDto, VerifyK1ImportDto, ConfirmK1ImportDto) in libs/common/src/lib/dtos/k1-import/ +- [X] T001 Install npm dependencies: pdf-parse, @azure/ai-form-recognizer, tesseract.js, @types/pdf-parse in package.json +- [X] T002 [P] Add K1ImportStatus enum (PROCESSING, EXTRACTED, VERIFIED, CONFIRMED, CANCELLED, FAILED), K1ImportSession model, CellMapping model, and CellAggregationRule model to prisma/schema.prisma +- [X] T003 [P] Create shared K-1 TypeScript interfaces (K1ExtractionResult, K1ExtractedField, K1UnmappedItem, K1ConfirmationRequest) in libs/common/src/lib/interfaces/k1-import.interface.ts +- [X] T004 [P] Register AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT and AZURE_DOCUMENT_INTELLIGENCE_KEY environment variables in apps/api/src/app/configuration/configuration.service.ts +- [X] T005 Run Prisma migration to create K-1 import tables and K1ImportStatus enum +- [X] T006 [P] Create shared DTOs (CreateK1ImportDto, VerifyK1ImportDto, ConfirmK1ImportDto) in libs/common/src/lib/dtos/k1-import/ ---