From f427fc18310bbe7334154f79292ea4fa5c50a635 Mon Sep 17 00:00:00 2001 From: nic Date: Fri, 20 Jun 2025 13:08:09 -0400 Subject: [PATCH] fix: removal now works on type filters and a combination of all filters --- internal/api/attachments.go | 101 ++++++++++++++ internal/handlers/web/document_remove.go | 160 +++++++++++++++++++++-- static/css/upload.css | 5 +- templates/partials/document_upload.html | 92 +++++++++++-- 4 files changed, 334 insertions(+), 24 deletions(-) diff --git a/internal/api/attachments.go b/internal/api/attachments.go index 237a762..4bd62c3 100644 --- a/internal/api/attachments.go +++ b/internal/api/attachments.go @@ -480,3 +480,104 @@ func (s *Session) UploadAttachmentFile(jobID, filename, purpose string, fileRead log.Printf("Successfully uploaded streaming attachment %s to job %s", filename, jobID) return result, nil } + +// GetJobAttachmentsDirect tries to get attachments directly using the attachment endpoint +func (s *Session) GetJobAttachmentsDirect(jobID string) ([]map[string]interface{}, error) { + log.Printf("GetJobAttachmentsDirect: Fetching attachments directly for job %s", jobID) + + // Try to get attachments using the attachment endpoint with entityId filter + url := fmt.Sprintf("%s/attachment", BaseURL) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + + // Add authorization header + req.Header.Set("Cookie", s.Cookie) + + // Add query parameters to filter by job + q := req.URL.Query() + q.Add("entityType", "3") // 3 = Job + q.Add("entityId", jobID) + req.URL.RawQuery = q.Encode() + + log.Printf("GetJobAttachmentsDirect: Using URL: %s", req.URL.String()) + + // Send the request + resp, err := s.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("error sending request: %v", err) + } + defer resp.Body.Close() + + // Read the response + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + // Check for errors + if resp.StatusCode != http.StatusOK { + log.Printf("GetJobAttachmentsDirect: API error (status %d) for job %s: %s", resp.StatusCode, jobID, string(body)) + return nil, fmt.Errorf("API returned error: %d %s - %s", resp.StatusCode, resp.Status, string(body)) + } + + // Log response preview + responsePreview := string(body) + if len(responsePreview) > 200 { + responsePreview = responsePreview[:200] + "... [truncated]" + } + log.Printf("GetJobAttachmentsDirect: Response preview for job %s: %s", jobID, responsePreview) + + // Parse the response + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("error parsing response: %v", err) + } + + // Try to extract attachments from various response structures + attachments := make([]map[string]interface{}, 0) + + // Check for data.attachments (this is what the API is actually returning) + if data, ok := result["data"].(map[string]interface{}); ok { + if attList, ok := data["attachments"].([]interface{}); ok { + log.Printf("GetJobAttachmentsDirect: Found %d attachments in data.attachments for job %s", len(attList), jobID) + for _, att := range attList { + if attachment, ok := att.(map[string]interface{}); ok { + attachments = append(attachments, attachment) + } + } + return attachments, nil + } + } + + // Check for objects array at root level + if objects, ok := result["objects"].([]interface{}); ok { + log.Printf("GetJobAttachmentsDirect: Found %d objects at root level for job %s", len(objects), jobID) + for _, obj := range objects { + if attachment, ok := obj.(map[string]interface{}); ok { + attachments = append(attachments, attachment) + } + } + return attachments, nil + } + + // Check for data.objects + if data, ok := result["data"].(map[string]interface{}); ok { + if objects, ok := data["objects"].([]interface{}); ok { + log.Printf("GetJobAttachmentsDirect: Found %d objects in data.objects for job %s", len(objects), jobID) + for _, obj := range objects { + if attachment, ok := obj.(map[string]interface{}); ok { + attachments = append(attachments, attachment) + } + } + return attachments, nil + } + } + + log.Printf("GetJobAttachmentsDirect: No attachments found for job %s", jobID) + return attachments, nil +} + +// Helper function to check if a string matches any pattern in a slice diff --git a/internal/handlers/web/document_remove.go b/internal/handlers/web/document_remove.go index f77b635..847a022 100644 --- a/internal/handlers/web/document_remove.go +++ b/internal/handlers/web/document_remove.go @@ -480,6 +480,8 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { if types := r.Form["documentTypes"]; len(types) > 0 { docTypes = types log.Printf("Filtering by document types: %v", docTypes) + } else { + log.Printf("No document types specified, will process all document types") } // Get filename patterns to match (optional) @@ -582,21 +584,54 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { // Get attachments using the most reliable method first var attachments []map[string]interface{} - // Try the specialized GetJobAttachments method first - attachments, err = session.GetJobAttachments(jobID) + // Try the direct attachment endpoint first - this should return ALL attachment types + directAttachments, err := session.GetJobAttachmentsDirect(jobID) if err != nil { - log.Printf("Error getting attachments for job %s: %v", jobID, err) + log.Printf("Error getting direct attachments for job %s: %v", jobID, err) attachments = []map[string]interface{}{} // Ensure it's initialized + } else { + attachments = directAttachments + if len(attachments) > 0 { + log.Printf("Found %d attachments via GetJobAttachmentsDirect for job %s", len(attachments), jobID) + } + } + + // If no attachments found via direct method, try the comprehensive GetAttachmentsForJob method + if len(attachments) == 0 { + jobData, err := session.GetAttachmentsForJob(jobID) + if err != nil { + log.Printf("Error getting attachments for job %s: %v", jobID, err) + attachments = []map[string]interface{}{} // Ensure it's initialized + } else { + // Extract attachments from the job data response + attachments = extractAttachmentsFromJobData(jobData, jobID) + if len(attachments) > 0 { + log.Printf("Found %d attachments via GetAttachmentsForJob for job %s", len(attachments), jobID) + } + } } - // If no attachments found, try the paperwork endpoint as fallback + // If still no attachments found, try the specialized GetJobAttachments method as fallback + if len(attachments) == 0 { + attachments, err = session.GetJobAttachments(jobID) + if err != nil { + log.Printf("Error getting attachments via GetJobAttachments for job %s: %v", jobID, err) + attachments = []map[string]interface{}{} // Ensure it's initialized + } else if len(attachments) > 0 { + log.Printf("Found %d attachments via GetJobAttachments for job %s", len(attachments), jobID) + } + } + + // If still no attachments found, try the paperwork endpoint as final fallback if len(attachments) == 0 { - log.Printf("No attachments found via GetJobAttachments for job %s, trying paperwork endpoint", jobID) paperworkItems, err := session.GetJobPaperwork(jobID) if err != nil { log.Printf("Error getting paperwork for job %s: %v", jobID, err) } else { attachments = paperworkItems + if len(attachments) > 0 { + log.Printf("Found %d attachments via GetJobPaperwork for job %s", len(attachments), jobID) + } } } @@ -624,6 +659,15 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { filename = fmt.Sprintf("Attachment ID: %s", attachmentIDStr) } + // Get purposeId for debugging + var purposeId float64 + if pid, ok := attachment["purposeId"].(float64); ok { + purposeId = pid + } + + // Track if this attachment should be included + includeAttachment := true + // Check document type filter if len(docTypes) > 0 { typeMatches := false @@ -639,17 +683,21 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { } } if !typeMatches { - continue + log.Printf("Job %s: Skipping '%s' (type %v) - no document type match", jobID, filename, purposeId) + includeAttachment = false } } - // Check filename pattern - if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) { - continue + // Check filename pattern filter + if includeAttachment && len(filenamePatterns) > 0 { + if !matchesAnyPattern(filename, filenamePatterns) { + log.Printf("Job %s: Skipping '%s' - no filename pattern match", jobID, filename) + includeAttachment = false + } } // Check age filter if applicable - if ageFilterDays > 0 { + if includeAttachment && ageFilterDays > 0 { var createdAt time.Time var hasDate bool @@ -670,14 +718,20 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { } if hasDate && createdAt.After(cutoffDate) { - continue // Skip if not old enough + log.Printf("Job %s: Skipping '%s' - not old enough (%s)", jobID, filename, createdAt.Format("2006-01-02")) + includeAttachment = false } } - // This attachment matches all criteria - attachmentsToDelete = append(attachmentsToDelete, attachment) + // Add attachment if it passed all filters + if includeAttachment { + log.Printf("Job %s: Including '%s' (type %v) for deletion", jobID, filename, purposeId) + attachmentsToDelete = append(attachmentsToDelete, attachment) + } } + log.Printf("Job %s: Found %d attachments to delete out of %d total attachments", jobID, len(attachmentsToDelete), len(attachments)) + jobResult.FilesFound = len(attachmentsToDelete) // Process deletions with rate limiting @@ -737,6 +791,8 @@ func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { deletionWg.Wait() + log.Printf("Job %s: Deletion completed. Files found: %d, Files removed: %d, Success: %v", jobID, jobResult.FilesFound, jobResult.FilesRemoved, jobResult.Success) + mu.Lock() results.JobsProcessed++ if jobResult.Success { @@ -1008,3 +1064,81 @@ func RemovalJobFileHandler(w http.ResponseWriter, r *http.Request) { return } } + +// Helper function to extract attachments from job data response +func extractAttachmentsFromJobData(jobData map[string]interface{}, jobID string) []map[string]interface{} { + attachments := make([]map[string]interface{}, 0) + + // Check if data key exists + if data, ok := jobData["data"].(map[string]interface{}); ok { + // Look for attachments directly in data + if attList, ok := data["attachments"].([]interface{}); ok && len(attList) > 0 { + for _, att := range attList { + if attachment, ok := att.(map[string]interface{}); ok { + attachments = append(attachments, attachment) + } + } + return attachments + } + + // Check for paperwork in data + if paperwork, ok := data["paperwork"].([]interface{}); ok && len(paperwork) > 0 { + for _, doc := range paperwork { + if docMap, ok := doc.(map[string]interface{}); ok { + attachments = append(attachments, docMap) + } + } + return attachments + } + + // Check for paperwork_objects (from fallback paperwork endpoint) + if paperworkObjects, ok := data["paperwork_objects"].([]interface{}); ok && len(paperworkObjects) > 0 { + for _, obj := range paperworkObjects { + if objMap, ok := obj.(map[string]interface{}); ok { + attachments = append(attachments, objMap) + } + } + return attachments + } + + // Check for other potential containers + for _, key := range []string{"objects", "components", "items", "records"} { + if items, ok := data[key].([]interface{}); ok && len(items) > 0 { + for _, item := range items { + if itemMap, ok := item.(map[string]interface{}); ok { + attachments = append(attachments, itemMap) + } + } + return attachments + } + } + } + + // Check for objects at root level + if objects, ok := jobData["objects"].([]interface{}); ok && len(objects) > 0 { + for _, obj := range objects { + if objMap, ok := obj.(map[string]interface{}); ok { + attachments = append(attachments, objMap) + } + } + return attachments + } + + return attachments +} + +// Helper function to get map keys +func getMapKeys(m interface{}) []string { + keys := make([]string, 0) + switch m := m.(type) { + case map[string]interface{}: + for k := range m { + keys = append(keys, k) + } + case map[string]string: + for k := range m { + keys = append(keys, k) + } + } + return keys +} diff --git a/static/css/upload.css b/static/css/upload.css index 83b4d76..e98f930 100644 --- a/static/css/upload.css +++ b/static/css/upload.css @@ -225,7 +225,7 @@ .modal { display: none; position: fixed; - z-index: 1001; + z-index: 1002; left: 0; top: 0; width: 100%; @@ -609,7 +609,7 @@ .form-actions { display: flex; gap: 1rem; - justify-content: flex-end; + justify-content: center; margin-top: 2rem; padding-top: 1rem; border-top: 1px solid var(--content-border); @@ -826,6 +826,7 @@ display: flex; gap: 0.5rem; align-items: center; + justify-content: center; } .file-pagination-buttons .pagination-btn { diff --git a/templates/partials/document_upload.html b/templates/partials/document_upload.html index ea76bfb..20e2089 100644 --- a/templates/partials/document_upload.html +++ b/templates/partials/document_upload.html @@ -134,6 +134,8 @@