From 54ecba5940a9ed1806f62909509218035c5a56ae Mon Sep 17 00:00:00 2001 From: nic Date: Thu, 19 Jun 2025 11:27:38 -0400 Subject: [PATCH] fix: updated document removal for better efficiency hopefully --- internal/handlers/web/document_remove.go | 903 ++--------------------- 1 file changed, 45 insertions(+), 858 deletions(-) diff --git a/internal/handlers/web/document_remove.go b/internal/handlers/web/document_remove.go index 3a0a0b2..dc67787 100644 --- a/internal/handlers/web/document_remove.go +++ b/internal/handlers/web/document_remove.go @@ -3,9 +3,7 @@ package web import ( "bytes" "encoding/csv" - "encoding/json" "fmt" - "io" "log" "net/http" "regexp" @@ -415,18 +413,6 @@ func renderErrorTemplate(w http.ResponseWriter, templateName, errorMsg string, j } } -// Helper function to get map keys as a string for logging -func mapKeysStr(m map[string]interface{}) string { - if m == nil { - return "nil" - } - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return strings.Join(keys, ", ") -} - // BulkRemoveDocumentsHandler handles bulk removal of documents from multiple jobs func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -516,7 +502,7 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { results := BulkRemovalResult{} var wg sync.WaitGroup var mu sync.Mutex - semaphore := make(chan struct{}, 5) // Limit concurrent API calls + semaphore := make(chan struct{}, 5) // Increased back to 5 since we've optimized the overhead // Process each job for _, jobID := range jobs { @@ -548,306 +534,11 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { } // Check job permissions first - log.Printf("**** JOB %s: Checking permissions...", jobID) hasAccess, reason, err := session.CheckJobPermissions(jobID) if err != nil { - log.Printf("**** JOB %s: Error checking permissions: %v", jobID, err) - } else { - log.Printf("**** JOB %s: Permission check result: access=%v, reason=%s", - jobID, hasAccess, reason) - } - - // Create the attachments array that will hold all found documents - var attachments []map[string]interface{} - - // Only proceed if we have access or couldn't determine access - if err != nil || hasAccess { - // Try job paperwork endpoint - if stringInSlice("1", docTypes) || len(docTypes) == 0 { - log.Printf("**** JOB %s: Trying the job paperwork endpoint", jobID) - paperworkURL := fmt.Sprintf("%s/job/%s/paperwork", api.BaseURL, jobID) - paperworkReq, err := http.NewRequest("GET", paperworkURL, nil) - if err == nil { - paperworkReq.Header.Set("Cookie", session.Cookie) - paperworkReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending request to job paperwork endpoint: %s", jobID, paperworkURL) - paperworkResp, err := session.Client.Do(paperworkReq) - - if err == nil && paperworkResp.StatusCode == http.StatusOK { - defer paperworkResp.Body.Close() - paperworkBody, _ := io.ReadAll(paperworkResp.Body) - - // Log preview of the response - responsePreview := string(paperworkBody) - if len(responsePreview) > 300 { - responsePreview = responsePreview[:300] + "... [truncated]" - } - log.Printf("**** JOB %s: Job paperwork response preview: %s", jobID, responsePreview) - - var paperworkResult map[string]interface{} - if err := json.Unmarshal(paperworkBody, &paperworkResult); err == nil { - // Process objects array if it exists - if objects, ok := paperworkResult["objects"].([]interface{}); ok && len(objects) > 0 { - log.Printf("**** JOB %s: Found %d paperwork items in objects array", jobID, len(objects)) - - for _, obj := range objects { - if paperworkMap, ok := obj.(map[string]interface{}); ok { - // Set purposeId to 1 for job paperwork - paperworkMap["purposeId"] = float64(1) - attachments = append(attachments, paperworkMap) - log.Printf("**** JOB %s: Added job paperwork to attachments", jobID) - } - } - } else if data, ok := paperworkResult["data"].(map[string]interface{}); ok { - // Check in data for attachments - if attachmentsArray, ok := data["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 { - log.Printf("**** JOB %s: Found %d paperwork items in data.attachments", jobID, len(attachmentsArray)) - - for _, att := range attachmentsArray { - if attMap, ok := att.(map[string]interface{}); ok { - // Ensure purposeId is set correctly - attMap["purposeId"] = float64(1) - attachments = append(attachments, attMap) - log.Printf("**** JOB %s: Added job paperwork from data.attachments", jobID) - } - } - } - - // Also check other locations in data - possibleKeys := []string{"paperwork", "objects"} - for _, key := range possibleKeys { - if items, ok := data[key].([]interface{}); ok && len(items) > 0 { - log.Printf("**** JOB %s: Found %d paperwork items in data.%s", jobID, len(items), key) - - for _, item := range items { - if itemMap, ok := item.(map[string]interface{}); ok { - // Set purposeId to 1 for job paperwork - itemMap["purposeId"] = float64(1) - attachments = append(attachments, itemMap) - log.Printf("**** JOB %s: Added job paperwork from data.%s", jobID, key) - } - } - } - } - } - } - } else { - log.Printf("**** JOB %s: Job paperwork endpoint failed or returned non-200 status: %v", jobID, err) - } - } - } - - // Try job invoice endpoint - if stringInSlice("14", docTypes) || len(docTypes) == 0 { - log.Printf("**** JOB %s: Trying the job invoice endpoint", jobID) - invoiceURL := fmt.Sprintf("%s/job/%s/invoice", api.BaseURL, jobID) - invoiceReq, err := http.NewRequest("GET", invoiceURL, nil) - if err == nil { - invoiceReq.Header.Set("Cookie", session.Cookie) - invoiceReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending request to job invoice endpoint: %s", jobID, invoiceURL) - invoiceResp, err := session.Client.Do(invoiceReq) - - if err == nil && invoiceResp.StatusCode == http.StatusOK { - defer invoiceResp.Body.Close() - invoiceBody, _ := io.ReadAll(invoiceResp.Body) - - // Log preview of the response - responsePreview := string(invoiceBody) - if len(responsePreview) > 300 { - responsePreview = responsePreview[:300] + "... [truncated]" - } - log.Printf("**** JOB %s: Job invoice response preview: %s", jobID, responsePreview) - - var invoiceResult map[string]interface{} - if err := json.Unmarshal(invoiceBody, &invoiceResult); err == nil { - // Process objects array if it exists - if objects, ok := invoiceResult["objects"].([]interface{}); ok && len(objects) > 0 { - log.Printf("**** JOB %s: Found %d job invoices in objects array", jobID, len(objects)) - - for _, obj := range objects { - if invoiceMap, ok := obj.(map[string]interface{}); ok { - // Set purposeId to 14 for job invoices - invoiceMap["purposeId"] = float64(14) - attachments = append(attachments, invoiceMap) - log.Printf("**** JOB %s: Added job invoice to attachments", jobID) - } - } - } else if data, ok := invoiceResult["data"].(map[string]interface{}); ok { - // Check in data for attachments - possibleKeys := []string{"invoices", "attachments", "objects"} - for _, key := range possibleKeys { - if items, ok := data[key].([]interface{}); ok && len(items) > 0 { - log.Printf("**** JOB %s: Found %d invoices in data.%s", jobID, len(items), key) - - for _, item := range items { - if itemMap, ok := item.(map[string]interface{}); ok { - // Set purposeId to 14 for job invoices - itemMap["purposeId"] = float64(14) - attachments = append(attachments, itemMap) - log.Printf("**** JOB %s: Added job invoice from data.%s", jobID, key) - } - } - } - } - } - } - } else { - log.Printf("**** JOB %s: Job invoice endpoint failed or returned non-200 status: %v", jobID, err) - } - } - } - - // Try generic attachment endpoint - if stringInSlice("7", docTypes) || len(docTypes) == 0 { - log.Printf("**** JOB %s: Trying the generic attachment endpoint", jobID) - genericURL := fmt.Sprintf("%s/job/%s/attachment", api.BaseURL, jobID) - genericReq, err := http.NewRequest("GET", genericURL, nil) - if err == nil { - genericReq.Header.Set("Cookie", session.Cookie) - genericReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending request to generic attachment endpoint: %s", jobID, genericURL) - genericResp, err := session.Client.Do(genericReq) - - if err == nil && genericResp.StatusCode == http.StatusOK { - defer genericResp.Body.Close() - genericBody, _ := io.ReadAll(genericResp.Body) - - // Log preview of the response - responsePreview := string(genericBody) - if len(responsePreview) > 300 { - responsePreview = responsePreview[:300] + "... [truncated]" - } - log.Printf("**** JOB %s: Generic attachment response preview: %s", jobID, responsePreview) - - var genericResult map[string]interface{} - if err := json.Unmarshal(genericBody, &genericResult); err == nil { - // Process objects array if it exists - if objects, ok := genericResult["objects"].([]interface{}); ok && len(objects) > 0 { - log.Printf("**** JOB %s: Found %d generic attachments in objects array", jobID, len(objects)) - - for _, obj := range objects { - if attachMap, ok := obj.(map[string]interface{}); ok { - // Set purposeId to 7 for generic attachments - attachMap["purposeId"] = float64(7) - attachments = append(attachments, attachMap) - log.Printf("**** JOB %s: Added generic attachment to attachments", jobID) - } - } - } else if data, ok := genericResult["data"].(map[string]interface{}); ok { - // Check in data for attachments - possibleKeys := []string{"attachments", "objects"} - for _, key := range possibleKeys { - if items, ok := data[key].([]interface{}); ok && len(items) > 0 { - log.Printf("**** JOB %s: Found %d generic attachments in data.%s", jobID, len(items), key) - - for _, item := range items { - if itemMap, ok := item.(map[string]interface{}); ok { - // Set purposeId to 7 for generic attachments - itemMap["purposeId"] = float64(7) - attachments = append(attachments, itemMap) - log.Printf("**** JOB %s: Added generic attachment from data.%s", jobID, key) - } - } - } - } - } - } - } else { - log.Printf("**** JOB %s: Generic attachment endpoint failed or returned non-200 status: %v", jobID, err) - } - } - } - - // Try vendor bill endpoint - if stringInSlice("2", docTypes) || len(docTypes) == 0 { - log.Printf("**** JOB %s: Trying the vendor invoice endpoint", jobID) - vendorInvoiceURL := fmt.Sprintf("%s/job/%s/vendorinvoice", api.BaseURL, jobID) - vendorInvoiceReq, err := http.NewRequest("GET", vendorInvoiceURL, nil) - if err == nil { - vendorInvoiceReq.Header.Set("Cookie", session.Cookie) - vendorInvoiceReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending request to vendor invoice endpoint: %s", jobID, vendorInvoiceURL) - vendorInvoiceResp, err := session.Client.Do(vendorInvoiceReq) - - if err == nil && vendorInvoiceResp.StatusCode == http.StatusOK { - defer vendorInvoiceResp.Body.Close() - vendorInvoiceBody, _ := io.ReadAll(vendorInvoiceResp.Body) - - // Log preview of the response - responsePreview := string(vendorInvoiceBody) - if len(responsePreview) > 300 { - responsePreview = responsePreview[:300] + "... [truncated]" - } - log.Printf("**** JOB %s: Vendor invoice response preview: %s", jobID, responsePreview) - - var vendorInvoiceResult map[string]interface{} - if err := json.Unmarshal(vendorInvoiceBody, &vendorInvoiceResult); err == nil { - // Process objects array if it exists - if objects, ok := vendorInvoiceResult["objects"].([]interface{}); ok && len(objects) > 0 { - log.Printf("**** JOB %s: Found %d vendor invoices in objects array", jobID, len(objects)) - - for _, obj := range objects { - if invoiceMap, ok := obj.(map[string]interface{}); ok { - // Set purposeId to 2 for vendor bills - invoiceMap["purposeId"] = float64(2) - attachments = append(attachments, invoiceMap) - log.Printf("**** JOB %s: Added vendor invoice to attachments", jobID) - } - } - } else if data, ok := vendorInvoiceResult["data"].(map[string]interface{}); ok { - // Check in data for attachments - possibleKeys := []string{"invoices", "vendorInvoices", "attachments", "objects"} - for _, key := range possibleKeys { - if items, ok := data[key].([]interface{}); ok && len(items) > 0 { - log.Printf("**** JOB %s: Found %d vendor bills in data.%s", jobID, len(items), key) - - for _, item := range items { - if itemMap, ok := item.(map[string]interface{}); ok { - // Set purposeId to 2 for vendor bills - itemMap["purposeId"] = float64(2) - attachments = append(attachments, itemMap) - log.Printf("**** JOB %s: Added vendor bill from data.%s", jobID, key) - } - } - } - } - } - } - } else { - log.Printf("**** JOB %s: Vendor invoice endpoint failed or returned non-200 status: %v", jobID, err) - } - } - } - - // Then continue with general paperwork endpoint to catch any we might have missed - log.Printf("**** JOB %s: Trying general paperwork endpoint", jobID) - - // Directly try to get paperwork using the specialized API - log.Printf("**** JOB %s: Using specialized paperwork API", jobID) - paperworkItems, err := session.GetJobPaperwork(jobID) - - if err != nil { - log.Printf("**** JOB %s: Error getting paperwork: %v", jobID, err) - } else if len(paperworkItems) > 0 { - log.Printf("**** JOB %s: GetJobPaperwork returned %d paperwork items", - jobID, len(paperworkItems)) - - // Add all paperwork items to attachments - for _, item := range paperworkItems { - log.Printf("**** JOB %s: Adding item from GetJobPaperwork: %v", - jobID, mapKeysStr(item)) - attachments = append(attachments, item) - } - } else { - log.Printf("**** JOB %s: No paperwork found using specialized API", jobID) - } - } else { - log.Printf("**** JOB %s: WARNING: No access to this job - reason: %s", jobID, reason) + log.Printf("Error checking permissions for job %s: %v", jobID, err) + } else if !hasAccess { + log.Printf("No access to job %s: %s", jobID, reason) mu.Lock() jobResult.Success = false jobResult.ErrorMsg = fmt.Sprintf("Cannot access job: %s", reason) @@ -857,432 +548,40 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { return } - // Try alternate method (always try both methods to see what data is available) - log.Printf("**** JOB %s: Retrieving attachments using GetAttachmentsForJob", jobID) - apiResponse, err := session.GetAttachmentsForJob(jobID) - if err != nil { - log.Printf("**** JOB %s: Error in GetAttachmentsForJob: %v", jobID, err) - } else { - // Log the structure of the response to understand format - rootKeys := make([]string, 0) - for k := range apiResponse { - rootKeys = append(rootKeys, k) - } - log.Printf("**** JOB %s: GetAttachmentsForJob returned response with root keys: %s", - jobID, strings.Join(rootKeys, ", ")) - - // Check if we have a data object - if data, ok := apiResponse["data"].(map[string]interface{}); ok { - dataKeys := make([]string, 0) - for k := range data { - dataKeys = append(dataKeys, k) - } - log.Printf("**** JOB %s: data object keys: %s", - jobID, strings.Join(dataKeys, ", ")) - } - - // Check if we have paperwork_data - if paperworkData, ok := apiResponse["paperwork_data"].(map[string]interface{}); ok { - dataKeys := make([]string, 0) - for k := range paperworkData { - dataKeys = append(dataKeys, k) - } - log.Printf("**** JOB %s: paperwork_data keys: %s", - jobID, strings.Join(dataKeys, ", ")) - - // Check if paperwork_data.data.attachments exists - if paperworkDataInner, ok := paperworkData["data"].(map[string]interface{}); ok { - if attachmentsArray, ok := paperworkDataInner["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 { - log.Printf("**** JOB %s: Found %d attachments in paperwork_data.data.attachments", - jobID, len(attachmentsArray)) - - // Process each attachment and add to our collection - for _, attachment := range attachmentsArray { - if attachmentMap, ok := attachment.(map[string]interface{}); ok { - attachments = append(attachments, attachmentMap) - } - } - } - } - } - - log.Printf("**** JOB %s: Total attachments gathered: %d", jobID, len(attachments)) - - if len(attachments) == 0 { - log.Printf("**** JOB %s: No attachments found yet, trying direct paperwork endpoint", jobID) - - // Directly try to get vendor bills using a specific endpoint - log.Printf("**** JOB %s: Trying vendor bill specific endpoint", jobID) - vendorBillURL := fmt.Sprintf("%s/job/%s/vendor-bill", api.BaseURL, jobID) - vendorBillReq, err := http.NewRequest("GET", vendorBillURL, nil) - if err == nil { - vendorBillReq.Header.Set("Cookie", session.Cookie) - vendorBillReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending request to vendor bill endpoint: %s", jobID, vendorBillURL) - vendorBillResp, err := session.Client.Do(vendorBillReq) - - if err == nil && vendorBillResp.StatusCode == http.StatusOK { - defer vendorBillResp.Body.Close() - vendorBillBody, _ := io.ReadAll(vendorBillResp.Body) - - // Log full response structure for debugging - log.Printf("**** JOB %s: Full vendor bill response: %s", jobID, string(vendorBillBody)) - - var vendorBillResult map[string]interface{} - if err := json.Unmarshal(vendorBillBody, &vendorBillResult); err == nil { - // Log all root keys in the response - rootKeys := make([]string, 0) - for k := range vendorBillResult { - rootKeys = append(rootKeys, k) - } - log.Printf("**** JOB %s: Vendor bill response root keys: %s", - jobID, strings.Join(rootKeys, ", ")) - - // Check if data exists and log all its keys - if data, ok := vendorBillResult["data"].(map[string]interface{}); ok { - dataKeys := make([]string, 0) - for k := range data { - dataKeys = append(dataKeys, k) - } - log.Printf("**** JOB %s: Vendor bill data keys: %s", - jobID, strings.Join(dataKeys, ", ")) - - // First try vendorBills directly - if vendorBills, ok := data["vendorBills"].([]interface{}); ok && len(vendorBills) > 0 { - log.Printf("**** JOB %s: Found %d vendor bills in data.vendorBills", jobID, len(vendorBills)) - - for _, bill := range vendorBills { - if billMap, ok := bill.(map[string]interface{}); ok { - // Set purposeId to 2 for vendor bills - billMap["purposeId"] = float64(2) - attachments = append(attachments, billMap) - } - } - } else { - // Try other possible locations - log.Printf("**** JOB %s: No vendorBills found in data, checking other locations", jobID) - - // Try each possible location for the vendor bills - possibleKeys := []string{"objects", "attachments", "bills", "paperwork", "documents"} - for _, key := range possibleKeys { - if items, ok := data[key].([]interface{}); ok && len(items) > 0 { - log.Printf("**** JOB %s: Found %d items in data.%s", jobID, len(items), key) - - // Log the structure of the first item - if itemMap, ok := items[0].(map[string]interface{}); ok { - itemKeys := make([]string, 0) - for k := range itemMap { - itemKeys = append(itemKeys, k) - } - log.Printf("**** JOB %s: First item in data.%s has keys: %s", - jobID, key, strings.Join(itemKeys, ", ")) - - // Log the first item as JSON for inspection - if itemJSON, err := json.Marshal(itemMap); err == nil { - log.Printf("**** JOB %s: First item in data.%s: %s", - jobID, key, string(itemJSON)) - } - } - - // Add all items as attachments - for _, item := range items { - if itemMap, ok := item.(map[string]interface{}); ok { - // Set purposeId to 2 for vendor bills - itemMap["purposeId"] = float64(2) - attachments = append(attachments, itemMap) - log.Printf("**** JOB %s: Added vendor bill from data.%s", jobID, key) - } - } - } - } - } - } else { - // If data is not a map, check for top-level objects - log.Printf("**** JOB %s: No data object in vendor bill response or it's not a map", jobID) - - if objects, ok := vendorBillResult["objects"].([]interface{}); ok && len(objects) > 0 { - log.Printf("**** JOB %s: Found %d objects at root level", jobID, len(objects)) - - for _, obj := range objects { - if objMap, ok := obj.(map[string]interface{}); ok { - // Set purposeId to 2 for vendor bills - objMap["purposeId"] = float64(2) - attachments = append(attachments, objMap) - log.Printf("**** JOB %s: Added vendor bill from root.objects", jobID) - } - } - } - } - } - } else { - log.Printf("**** JOB %s: Vendor bill endpoint failed or returned non-200 status: %v", jobID, err) - } - } - - // Also try direct paperwork endpoint - log.Printf("**** JOB %s: Trying direct paperwork endpoint", jobID) - paperworkURL := fmt.Sprintf("%s/job/%s/paperwork", api.BaseURL, jobID) - paperworkReq, err := http.NewRequest("GET", paperworkURL, nil) - if err == nil { - paperworkReq.Header.Set("Cookie", session.Cookie) - paperworkReq.Header.Set("Accept", "application/json") - - log.Printf("**** JOB %s: Sending direct request to %s", jobID, paperworkURL) - paperworkResp, err := session.Client.Do(paperworkReq) - - if err == nil && paperworkResp.StatusCode == http.StatusOK { - defer paperworkResp.Body.Close() - paperworkBody, _ := io.ReadAll(paperworkResp.Body) - - // Log preview of the response - responsePreview := string(paperworkBody) - if len(responsePreview) > 200 { - responsePreview = responsePreview[:200] + "... [truncated]" - } - log.Printf("**** JOB %s: Direct paperwork response preview: %s", jobID, responsePreview) - - // Parse the response - var paperworkResult map[string]interface{} - if err := json.Unmarshal(paperworkBody, &paperworkResult); err == nil { - // Log the structure of the response - rootKeys := make([]string, 0) - for k := range paperworkResult { - rootKeys = append(rootKeys, k) - } - log.Printf("**** JOB %s: Direct paperwork response keys: %s", - jobID, strings.Join(rootKeys, ", ")) - - // Check for data.attachments - if data, ok := paperworkResult["data"].(map[string]interface{}); ok { - dataKeys := make([]string, 0) - for k := range data { - dataKeys = append(dataKeys, k) - } - log.Printf("**** JOB %s: Direct paperwork data keys: %s", - jobID, strings.Join(dataKeys, ", ")) - - // Extract attachments from data.attachments - if attachmentsArray, ok := data["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 { - log.Printf("**** JOB %s: Found %d attachments in direct paperwork response", - jobID, len(attachmentsArray)) - - // Loop through the attachments and add to our collection - for i, attachment := range attachmentsArray { - if attachmentMap, ok := attachment.(map[string]interface{}); ok { - // Log details of the first attachment to understand the structure - if i == 0 { - log.Printf("**** JOB %s: First attachment structure: %+v", jobID, attachmentMap) - attKeys := make([]string, 0) - for k := range attachmentMap { - attKeys = append(attKeys, k) - } - log.Printf("**** JOB %s: First attachment keys: %s", jobID, strings.Join(attKeys, ", ")) - } - - // Add to our attachments collection - attachments = append(attachments, attachmentMap) - } - } - } - } - } - } - } - - // Log attachment count after direct endpoint - log.Printf("**** JOB %s: Attachments found after direct paperwork call: %d", jobID, len(attachments)) - - // Deduplicate attachments to avoid processing the same ones multiple times - originalCount := len(attachments) - attachments = deduplicateAttachments(attachments) - if len(attachments) < originalCount { - log.Printf("**** JOB %s: Removed %d duplicate attachments, %d unique attachments remain", - jobID, originalCount-len(attachments), len(attachments)) - } - - // Now actually apply the filters - filteredAttachments := make([]map[string]interface{}, 0) - - // Process each attachment - for _, attachment := range attachments { - log.Printf("**** JOB %s: Processing attachment ID: %v", jobID, attachment["id"]) - - // Check document types filter - if len(docTypes) > 0 { - typeMatches := false - - // Log all attachment details for debugging - logAttachmentDetails(jobID, attachment) - - // Log docTypes array as it comes from the form - log.Printf("**** JOB %s: Doc types from form: %v", jobID, docTypes) - - // Get all possible attachment type info - var purposeId float64 - var purpose string - var typeValue string - - // Check all possible type fields - if val, ok := attachment["purposeId"].(float64); ok { - purposeId = val - log.Printf("**** JOB %s: Found purposeId=%.0f", jobID, purposeId) - } - - if val, ok := attachment["purpose"].(string); ok { - purpose = val - log.Printf("**** JOB %s: Found purpose=%s", jobID, purpose) - } - - if val, ok := attachment["type"].(string); ok { - typeValue = val - log.Printf("**** JOB %s: Found type=%s", jobID, typeValue) - } - - // Now try to match with each document type from form - for _, docType := range docTypes { - // Clean up the doc type (remove leading zeros) - docTypeClean := strings.TrimLeft(docType, "0") - - // Try to convert the cleaned doc type to a number - if docTypeNum, err := strconv.ParseFloat(docTypeClean, 64); err == nil { - // Compare with purposeId - if purposeId > 0 && purposeId == docTypeNum { - log.Printf("**** JOB %s: MATCH! docType=%s matches purposeId=%.0f", - jobID, docType, purposeId) - typeMatches = true - break - } - } - - // Try string comparisons if no match yet - if !typeMatches && purpose != "" { - if docType == purpose || docTypeClean == purpose { - log.Printf("**** JOB %s: MATCH! docType=%s matches purpose=%s", - jobID, docType, purpose) - typeMatches = true - break - } - } - - if !typeMatches && typeValue != "" { - if docType == typeValue || docTypeClean == typeValue { - log.Printf("**** JOB %s: MATCH! docType=%s matches type=%s", - jobID, docType, typeValue) - typeMatches = true - break - } - } - } - - if !typeMatches { - log.Printf("**** JOB %s: No type match found, skipping attachment", jobID) - continue - } - } - - // Get filename from any available field, don't assume description exists - var filename string - if desc, ok := attachment["description"].(string); ok && desc != "" { - filename = desc - } else if name, ok := attachment["name"].(string); ok && name != "" { - filename = name - } else if fname, ok := attachment["fileName"].(string); ok && fname != "" { - filename = fname - } else { - // If no name is available, use the ID as the name - filename = fmt.Sprintf("Attachment ID: %s", attachment["id"]) - } - - log.Printf("**** JOB %s: Attachment filename: %s", jobID, filename) - - // Check filename pattern - if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) { - log.Printf("**** JOB %s: Skipping attachment - filename '%s' doesn't match patterns", - jobID, filename) - continue - } - - // Check age filter if applicable - if ageFilterDays > 0 { - // Get creation time - var createdAt time.Time - var createdOn string - var hasDate bool - - // Try to get the creation date - if created, ok := attachment["createdOn"].(string); ok { - createdOn = created - hasDate = true - } else if created, ok := attachment["created"].(string); ok { - createdOn = created - hasDate = true - } else if lastModified, ok := attachment["lastModified"].(string); ok { - createdOn = lastModified - hasDate = true - } else if createdVal, ok := attachment["created"].(float64); ok { - createdAt = time.Unix(int64(createdVal), 0) - createdOn = createdAt.Format(time.RFC3339) - hasDate = true - } - - if hasDate { - if parsedTime, err := time.Parse(time.RFC3339, createdOn); err == nil { - createdAt = parsedTime - if createdAt.After(cutoffDate) { - log.Printf("Skipping attachment %s - created on %s is newer than cutoff %s", - filename, createdAt.Format("2006-01-02"), cutoffDate.Format("2006-01-02")) - continue // Skip if not old enough - } - } - } - } - - // Use a new variable with a different name for the log message - var typeStr string = "unknown" - if pId, ok := attachment["purposeId"].(float64); ok { - typeStr = fmt.Sprintf("%.0f", pId) - } - - log.Printf("Attachment %s (type: %s) matches all criteria - queued for deletion", - filename, typeStr) - - // If we got here, the attachment passes all filters - filteredAttachments = append(filteredAttachments, attachment) - } - - // Update the attachments with the filtered list - attachments = filteredAttachments - log.Printf("**** JOB %s: Final attachments to process after filtering: %d", jobID, len(attachments)) - } + // Get attachments using the most reliable method first + var attachments []map[string]interface{} - log.Printf("**** JOB %s: Final total attachments: %d", jobID, len(attachments)) + // Try the specialized GetJobAttachments method first + attachments, err = session.GetJobAttachments(jobID) + if err != nil { + log.Printf("Error getting attachments for job %s: %v", jobID, err) + attachments = []map[string]interface{}{} // Ensure it's initialized + } - if len(attachments) == 0 { - log.Printf("**** JOB %s: WARNING! No attachments found after all retrieval attempts", jobID) + // If no attachments found, try the paperwork endpoint as fallback + if len(attachments) == 0 { + log.Printf("No attachments found via GetJobAttachments for job %s, trying paperwork endpoint", jobID) + paperworkItems, err := session.GetJobPaperwork(jobID) + if err != nil { + log.Printf("Error getting paperwork for job %s: %v", jobID, err) } else { - // Deduplicate again before continuing with deletion, in case multiple methods found the same attachments - originalCount := len(attachments) - attachments = deduplicateAttachments(attachments) - if len(attachments) < originalCount { - log.Printf("**** JOB %s: Final deduplication removed %d duplicates, %d unique attachments remain", - jobID, originalCount-len(attachments), len(attachments)) - } + attachments = paperworkItems } } + log.Printf("Found %d attachments for job %s", len(attachments), jobID) + // Filter attachments based on criteria var attachmentsToDelete []map[string]interface{} for _, attachment := range attachments { // Get the attachment ID attachmentIDRaw, idOk := attachment["id"].(float64) if !idOk { - log.Printf("Skipping attachment - missing ID field") continue } attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDRaw) - // Safely get filename from any available field + // Get filename from any available field var filename string if desc, ok := attachment["description"].(string); ok && desc != "" { filename = desc @@ -1291,105 +590,59 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { } else if fname, ok := attachment["fileName"].(string); ok && fname != "" { filename = fname } else { - // If no name is available, use the ID as the name filename = fmt.Sprintf("Attachment ID: %s", attachmentIDStr) } - if filename == "" { - log.Printf("Attachment %s is missing filename information", attachmentIDStr) - continue - } - - log.Printf("Processing attachment ID: %s, filename: %s", attachmentIDStr, filename) - - // Check document type using purposeId which IS available in the data + // Check document type filter if len(docTypes) > 0 { typeMatches := false - - // Get purposeId - var purposeId float64 - if val, ok := attachment["purposeId"].(float64); ok { - purposeId = val - log.Printf("Attachment %s has purposeId=%.0f", attachmentIDStr, purposeId) - } else { - log.Printf("Attachment %s has no purposeId field", attachmentIDStr) - continue - } - - // Compare with selected document types - for _, docType := range docTypes { - // Form uses "01", "02", etc. but API uses 1, 2, etc. - handle both - docTypeClean := strings.TrimLeft(docType, "0") - if docTypeInt, err := strconv.Atoi(docTypeClean); err == nil { - if float64(docTypeInt) == purposeId { - log.Printf("✓ Type match for attachment %s: form value %s matches purposeId %.0f", - attachmentIDStr, docType, purposeId) - typeMatches = true - break + if purposeId, ok := attachment["purposeId"].(float64); ok { + for _, docType := range docTypes { + docTypeClean := strings.TrimLeft(docType, "0") + if docTypeInt, err := strconv.Atoi(docTypeClean); err == nil { + if float64(docTypeInt) == purposeId { + typeMatches = true + break + } } } } - if !typeMatches { - // Get purposeId for error message - var purposeVal float64 - if val, ok := attachment["purposeId"].(float64); ok { - purposeVal = val - log.Printf("Skipping attachment %s - purposeId %.0f doesn't match any selected types: %v", - attachmentIDStr, purposeVal, docTypes) - } else { - log.Printf("Skipping attachment %s - type doesn't match any selected types: %v", - attachmentIDStr, docTypes) - } continue } } // Check filename pattern if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) { - log.Printf("Skipping attachment %s - filename '%s' doesn't match patterns: %v", - attachmentIDStr, filename, filenamePatterns) continue } // Check age filter if applicable if ageFilterDays > 0 { - // Get creation time var createdAt time.Time - var createdOn string var hasDate bool // Try to get the creation date if created, ok := attachment["createdOn"].(string); ok { - createdOn = created - hasDate = true + if parsedTime, err := time.Parse(time.RFC3339, created); err == nil { + createdAt = parsedTime + hasDate = true + } } else if created, ok := attachment["created"].(string); ok { - createdOn = created - hasDate = true - } else if lastModified, ok := attachment["lastModified"].(string); ok { - createdOn = lastModified - hasDate = true + if parsedTime, err := time.Parse(time.RFC3339, created); err == nil { + createdAt = parsedTime + hasDate = true + } } else if createdVal, ok := attachment["created"].(float64); ok { createdAt = time.Unix(int64(createdVal), 0) - createdOn = createdAt.Format(time.RFC3339) hasDate = true } - if hasDate { - if parsedTime, err := time.Parse(time.RFC3339, createdOn); err == nil { - createdAt = parsedTime - if createdAt.After(cutoffDate) { - log.Printf("Skipping attachment %s - created on %s is newer than cutoff %s", - filename, createdAt.Format("2006-01-02"), cutoffDate.Format("2006-01-02")) - continue // Skip if not old enough - } - } + if hasDate && createdAt.After(cutoffDate) { + continue // Skip if not old enough } } - // Log that we found an attachment to delete - log.Printf("Attachment %s matches criteria - will be deleted", filename) - // This attachment matches all criteria attachmentsToDelete = append(attachmentsToDelete, attachment) } @@ -1399,24 +652,18 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { // Process deletions with rate limiting var deletionWg sync.WaitGroup for _, attachment := range attachmentsToDelete { - // Use a separate goroutine for each deletion with its own semaphore slot deletionWg.Add(1) - - // Important: Create a copy of the attachment for the goroutine to avoid - // sharing the loop variable which can cause race conditions attachmentCopy := attachment go func(att map[string]interface{}) { defer deletionWg.Done() - // Acquire a semaphore slot for this deletion operation semaphore <- struct{}{} - defer func() { <-semaphore }() // Release when done + defer func() { <-semaphore }() attachmentIDFloat := att["id"].(float64) - attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDFloat) // Convert to string without decimal + attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDFloat) - // Safely get filename from any available field var filename string if desc, ok := att["description"].(string); ok && desc != "" { filename = desc @@ -1425,7 +672,6 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { } else if fname, ok := att["fileName"].(string); ok && fname != "" { filename = fname } else { - // If no name is available, use the ID as the name filename = fmt.Sprintf("Attachment ID: %s", attachmentIDStr) } @@ -1437,9 +683,6 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { Name: filename, } - // For all attachment types, we'll use the attachment endpoint for deletion - // The API endpoint is /attachment/{id} as defined in DeleteAttachment method - log.Printf("Deleting attachment %s (ID: %s) using attachment endpoint", filename, attachmentIDStr) deleteErr := session.DeleteAttachment(attachmentIDStr) mu.Lock() @@ -1448,22 +691,19 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { if deleteErr != nil { fileResult.Success = false fileResult.Error = deleteErr.Error() - log.Printf("Error deleting attachment %s: %v", filename, deleteErr) jobResult.Success = false } else { fileResult.Success = true jobResult.FilesRemoved++ - log.Printf("Successfully deleted attachment %s", filename) } jobResult.Files = append(jobResult.Files, fileResult) - // Add a slight delay to avoid overwhelming the API - time.Sleep(300 * time.Millisecond) + // Reduced delay + time.Sleep(200 * time.Millisecond) }(attachmentCopy) } - // Wait for all deletions for this job to complete deletionWg.Wait() mu.Lock() @@ -1586,16 +826,6 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { w.Write(resultHTML.Bytes()) } -// Helper function to check if a string is in a slice -func stringInSlice(s string, slice []string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - // Helper function to check if a string matches any pattern in a slice func matchesAnyPattern(s string, patterns []string) bool { // Convert the string to lowercase for case-insensitive comparison @@ -1619,46 +849,3 @@ func matchesAnyPattern(s string, patterns []string) bool { } return false } - -// Enhanced debugging function to help understand attachment structure -func logAttachmentDetails(jobID string, attachment map[string]interface{}) { - // Create a detailed view of the attachment - attachmentID := "unknown" - if id, ok := attachment["id"].(float64); ok { - attachmentID = fmt.Sprintf("%.0f", id) - } - - log.Printf("***** JOB %s - DETAILED ATTACHMENT %s *****", jobID, attachmentID) - for key, value := range attachment { - log.Printf(" JOB %s: %s = %v (type: %T)", jobID, key, value, value) - } - log.Printf("***** JOB %s - END ATTACHMENT DETAILS *****", jobID) -} - -// Helper function to deduplicate attachments based on ID -func deduplicateAttachments(attachments []map[string]interface{}) []map[string]interface{} { - seen := make(map[string]bool) - uniqueAttachments := make([]map[string]interface{}, 0) - - for _, attachment := range attachments { - // Get the ID as a string for deduplication - var idStr string - if id, ok := attachment["id"].(float64); ok { - idStr = fmt.Sprintf("%.0f", id) - } else if id, ok := attachment["id"].(string); ok { - idStr = id - } else { - // If no valid ID, just add it (should not happen) - uniqueAttachments = append(uniqueAttachments, attachment) - continue - } - - // Only add if we haven't seen this ID before - if !seen[idStr] { - seen[idStr] = true - uniqueAttachments = append(uniqueAttachments, attachment) - } - } - - return uniqueAttachments -}