|
|
|
@ -3,9 +3,13 @@ package web |
|
|
|
import ( |
|
|
|
"bytes" |
|
|
|
"encoding/csv" |
|
|
|
"encoding/json" |
|
|
|
"fmt" |
|
|
|
"io" |
|
|
|
"log" |
|
|
|
"net/http" |
|
|
|
"regexp" |
|
|
|
"sort" |
|
|
|
"strconv" |
|
|
|
"strings" |
|
|
|
"sync" |
|
|
|
@ -29,6 +33,13 @@ func DocumentRemoveHandler(w http.ResponseWriter, r *http.Request) { |
|
|
|
data := map[string]interface{}{ |
|
|
|
"Title": "Document Removal", |
|
|
|
"Session": session, |
|
|
|
"DocumentTypes": []map[string]string{ |
|
|
|
{"value": "1", "label": "Job Paperwork"}, |
|
|
|
{"value": "2", "label": "Job Vendor Bill"}, |
|
|
|
{"value": "4", "label": "Generic Attachment"}, |
|
|
|
{"value": "7", "label": "Blank Paperwork"}, |
|
|
|
{"value": "14", "label": "Job Invoice"}, |
|
|
|
}, |
|
|
|
} |
|
|
|
|
|
|
|
if r.Header.Get("HX-Request") == "true" { |
|
|
|
@ -406,3 +417,760 @@ func renderErrorTemplate(w http.ResponseWriter, templateName, errorMsg string, j |
|
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Helper function to get map keys as a string for logging
|
|
|
|
func mapKeysStr(m map[string]interface{}) string { |
|
|
|
if m == nil { |
|
|
|
return "nil" |
|
|
|
} |
|
|
|
keys := make([]string, 0, len(m)) |
|
|
|
for k := range m { |
|
|
|
keys = append(keys, k) |
|
|
|
} |
|
|
|
return strings.Join(keys, ", ") |
|
|
|
} |
|
|
|
|
|
|
|
// BulkRemoveDocumentsHandler handles bulk removal of documents from multiple jobs
|
|
|
|
func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) { |
|
|
|
if r.Method != http.MethodPost { |
|
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
session, ok := r.Context().Value("session").(*api.Session) |
|
|
|
if !ok { |
|
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
// Parse the form
|
|
|
|
if err := r.ParseForm(); err != nil { |
|
|
|
http.Error(w, fmt.Sprintf("Error parsing form: %v", err), http.StatusBadRequest) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
// Get job IDs from the form
|
|
|
|
jobIDs := r.FormValue("jobIDs") |
|
|
|
if jobIDs == "" { |
|
|
|
http.Error(w, "No job IDs provided", http.StatusBadRequest) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
jobs := strings.Split(jobIDs, ",") |
|
|
|
if len(jobs) == 0 { |
|
|
|
http.Error(w, "No valid job IDs provided", http.StatusBadRequest) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
// Get document types to remove (optional)
|
|
|
|
var docTypes []string |
|
|
|
if types := r.Form["documentTypes"]; len(types) > 0 { |
|
|
|
docTypes = types |
|
|
|
log.Printf("Filtering by document types: %v", docTypes) |
|
|
|
} |
|
|
|
|
|
|
|
// Get filename patterns to match (optional)
|
|
|
|
var filenamePatterns []string |
|
|
|
if patterns := r.FormValue("filenamePatterns"); patterns != "" { |
|
|
|
filenamePatterns = strings.Split(patterns, ",") |
|
|
|
for i, p := range filenamePatterns { |
|
|
|
filenamePatterns[i] = strings.TrimSpace(p) |
|
|
|
} |
|
|
|
log.Printf("Filtering by filename patterns: %v", filenamePatterns) |
|
|
|
} |
|
|
|
|
|
|
|
// Get age filter (optional)
|
|
|
|
var ageFilterDays int |
|
|
|
if ageStr := r.FormValue("ageFilter"); ageStr != "" { |
|
|
|
if days, err := strconv.Atoi(ageStr); err == nil && days > 0 { |
|
|
|
ageFilterDays = days |
|
|
|
log.Printf("Using age filter: older than %d days", ageFilterDays) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Calculate cutoff date if using age filter
|
|
|
|
var cutoffDate time.Time |
|
|
|
if ageFilterDays > 0 { |
|
|
|
cutoffDate = time.Now().AddDate(0, 0, -ageFilterDays) |
|
|
|
} |
|
|
|
|
|
|
|
// Structure to track results
|
|
|
|
type BulkRemovalResult struct { |
|
|
|
JobsProcessed int |
|
|
|
JobsWithErrors int |
|
|
|
TotalFiles int |
|
|
|
SuccessCount int |
|
|
|
ErrorCount int |
|
|
|
JobResults []struct { |
|
|
|
JobID string |
|
|
|
FilesFound int |
|
|
|
FilesRemoved int |
|
|
|
Success bool |
|
|
|
ErrorMsg string |
|
|
|
Files []struct { |
|
|
|
Name string |
|
|
|
Success bool |
|
|
|
Error string |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
results := BulkRemovalResult{} |
|
|
|
var wg sync.WaitGroup |
|
|
|
var mu sync.Mutex |
|
|
|
semaphore := make(chan struct{}, 5) // Limit concurrent API calls
|
|
|
|
|
|
|
|
// Process each job
|
|
|
|
for _, jobID := range jobs { |
|
|
|
wg.Add(1) |
|
|
|
go func(jobID string) { |
|
|
|
defer wg.Done() |
|
|
|
|
|
|
|
log.Printf("Processing job ID: %s for document removal", jobID) |
|
|
|
|
|
|
|
jobResult := struct { |
|
|
|
JobID string |
|
|
|
FilesFound int |
|
|
|
FilesRemoved int |
|
|
|
Success bool |
|
|
|
ErrorMsg string |
|
|
|
Files []struct { |
|
|
|
Name string |
|
|
|
Success bool |
|
|
|
Error string |
|
|
|
} |
|
|
|
}{ |
|
|
|
JobID: jobID, |
|
|
|
Success: true, |
|
|
|
Files: []struct { |
|
|
|
Name string |
|
|
|
Success bool |
|
|
|
Error string |
|
|
|
}{}, |
|
|
|
} |
|
|
|
|
|
|
|
// Check job permissions first
|
|
|
|
log.Printf("**** JOB %s: Checking permissions...", jobID) |
|
|
|
hasAccess, reason, err := session.CheckJobPermissions(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Printf("**** JOB %s: Error checking permissions: %v", jobID, err) |
|
|
|
} else { |
|
|
|
log.Printf("**** JOB %s: Permission check result: access=%v, reason=%s", |
|
|
|
jobID, hasAccess, reason) |
|
|
|
} |
|
|
|
|
|
|
|
// Create the attachments array that will hold all found documents
|
|
|
|
var attachments []map[string]interface{} |
|
|
|
|
|
|
|
// Only proceed if we have access or couldn't determine access
|
|
|
|
if err != nil || hasAccess { |
|
|
|
// Directly try to get paperwork using the specialized API
|
|
|
|
log.Printf("**** JOB %s: Using specialized paperwork API", jobID) |
|
|
|
paperworkItems, err := session.GetJobPaperwork(jobID) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
log.Printf("**** JOB %s: Error getting paperwork: %v", jobID, err) |
|
|
|
} else if len(paperworkItems) > 0 { |
|
|
|
log.Printf("**** JOB %s: GetJobPaperwork returned %d paperwork items", |
|
|
|
jobID, len(paperworkItems)) |
|
|
|
|
|
|
|
// Add all paperwork items to attachments
|
|
|
|
for _, item := range paperworkItems { |
|
|
|
log.Printf("**** JOB %s: Adding item from GetJobPaperwork: %v", |
|
|
|
jobID, mapKeysStr(item)) |
|
|
|
attachments = append(attachments, item) |
|
|
|
} |
|
|
|
} else { |
|
|
|
log.Printf("**** JOB %s: No paperwork found using specialized API", jobID) |
|
|
|
} |
|
|
|
} else { |
|
|
|
log.Printf("**** JOB %s: WARNING: No access to this job - reason: %s", jobID, reason) |
|
|
|
mu.Lock() |
|
|
|
jobResult.Success = false |
|
|
|
jobResult.ErrorMsg = fmt.Sprintf("Cannot access job: %s", reason) |
|
|
|
results.JobResults = append(results.JobResults, jobResult) |
|
|
|
results.JobsWithErrors++ |
|
|
|
mu.Unlock() |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
// Try alternate method (always try both methods to see what data is available)
|
|
|
|
log.Printf("**** JOB %s: Retrieving attachments using GetAttachmentsForJob", jobID) |
|
|
|
apiResponse, err := session.GetAttachmentsForJob(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Printf("**** JOB %s: Error in GetAttachmentsForJob: %v", jobID, err) |
|
|
|
} else { |
|
|
|
// Log the structure of the response to understand format
|
|
|
|
rootKeys := make([]string, 0) |
|
|
|
for k := range apiResponse { |
|
|
|
rootKeys = append(rootKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: GetAttachmentsForJob returned response with root keys: %s", |
|
|
|
jobID, strings.Join(rootKeys, ", ")) |
|
|
|
|
|
|
|
// Check if we have a data object
|
|
|
|
if data, ok := apiResponse["data"].(map[string]interface{}); ok { |
|
|
|
dataKeys := make([]string, 0) |
|
|
|
for k := range data { |
|
|
|
dataKeys = append(dataKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: data object keys: %s", |
|
|
|
jobID, strings.Join(dataKeys, ", ")) |
|
|
|
} |
|
|
|
|
|
|
|
// Check if we have paperwork_data
|
|
|
|
if paperworkData, ok := apiResponse["paperwork_data"].(map[string]interface{}); ok { |
|
|
|
dataKeys := make([]string, 0) |
|
|
|
for k := range paperworkData { |
|
|
|
dataKeys = append(dataKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: paperwork_data keys: %s", |
|
|
|
jobID, strings.Join(dataKeys, ", ")) |
|
|
|
|
|
|
|
// Check if paperwork_data.data.attachments exists
|
|
|
|
if paperworkDataInner, ok := paperworkData["data"].(map[string]interface{}); ok { |
|
|
|
if attachmentsArray, ok := paperworkDataInner["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 { |
|
|
|
log.Printf("**** JOB %s: Found %d attachments in paperwork_data.data.attachments", |
|
|
|
jobID, len(attachmentsArray)) |
|
|
|
|
|
|
|
// Process each attachment and add to our collection
|
|
|
|
for _, attachment := range attachmentsArray { |
|
|
|
if attachmentMap, ok := attachment.(map[string]interface{}); ok { |
|
|
|
attachments = append(attachments, attachmentMap) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("**** JOB %s: Total attachments gathered: %d", jobID, len(attachments)) |
|
|
|
|
|
|
|
if len(attachments) == 0 { |
|
|
|
log.Printf("**** JOB %s: No attachments found yet, trying direct paperwork endpoint") |
|
|
|
|
|
|
|
// Make a direct call to the paperwork endpoint
|
|
|
|
paperworkURL := fmt.Sprintf("%s/job/%s/paperwork", api.BaseURL, jobID) |
|
|
|
paperworkReq, err := http.NewRequest("GET", paperworkURL, nil) |
|
|
|
if err == nil { |
|
|
|
paperworkReq.Header.Set("Cookie", session.Cookie) |
|
|
|
paperworkReq.Header.Set("Accept", "application/json") |
|
|
|
|
|
|
|
log.Printf("**** JOB %s: Sending direct request to %s", jobID, paperworkURL) |
|
|
|
paperworkResp, err := session.Client.Do(paperworkReq) |
|
|
|
|
|
|
|
if err == nil && paperworkResp.StatusCode == http.StatusOK { |
|
|
|
defer paperworkResp.Body.Close() |
|
|
|
paperworkBody, _ := io.ReadAll(paperworkResp.Body) |
|
|
|
|
|
|
|
// Log preview of the response
|
|
|
|
responsePreview := string(paperworkBody) |
|
|
|
if len(responsePreview) > 200 { |
|
|
|
responsePreview = responsePreview[:200] + "... [truncated]" |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: Direct paperwork response preview: %s", jobID, responsePreview) |
|
|
|
|
|
|
|
// Parse the response
|
|
|
|
var paperworkResult map[string]interface{} |
|
|
|
if err := json.Unmarshal(paperworkBody, &paperworkResult); err == nil { |
|
|
|
// Log the structure of the response
|
|
|
|
rootKeys := make([]string, 0) |
|
|
|
for k := range paperworkResult { |
|
|
|
rootKeys = append(rootKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: Direct paperwork response keys: %s", |
|
|
|
jobID, strings.Join(rootKeys, ", ")) |
|
|
|
|
|
|
|
// Check for data.attachments
|
|
|
|
if data, ok := paperworkResult["data"].(map[string]interface{}); ok { |
|
|
|
dataKeys := make([]string, 0) |
|
|
|
for k := range data { |
|
|
|
dataKeys = append(dataKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: Direct paperwork data keys: %s", |
|
|
|
jobID, strings.Join(dataKeys, ", ")) |
|
|
|
|
|
|
|
// Extract attachments from data.attachments
|
|
|
|
if attachmentsArray, ok := data["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 { |
|
|
|
log.Printf("**** JOB %s: Found %d attachments in direct paperwork response", |
|
|
|
jobID, len(attachmentsArray)) |
|
|
|
|
|
|
|
// Loop through the attachments and add to our collection
|
|
|
|
for i, attachment := range attachmentsArray { |
|
|
|
if attachmentMap, ok := attachment.(map[string]interface{}); ok { |
|
|
|
// Log details of the first attachment to understand the structure
|
|
|
|
if i == 0 { |
|
|
|
log.Printf("**** JOB %s: First attachment structure: %+v", jobID, attachmentMap) |
|
|
|
attKeys := make([]string, 0) |
|
|
|
for k := range attachmentMap { |
|
|
|
attKeys = append(attKeys, k) |
|
|
|
} |
|
|
|
log.Printf("**** JOB %s: First attachment keys: %s", jobID, strings.Join(attKeys, ", ")) |
|
|
|
} |
|
|
|
|
|
|
|
// Add to our attachments collection
|
|
|
|
attachments = append(attachments, attachmentMap) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Log attachment count after direct endpoint
|
|
|
|
log.Printf("**** JOB %s: Attachments found after direct paperwork call: %d", jobID, len(attachments)) |
|
|
|
|
|
|
|
// Now actually apply the filters
|
|
|
|
filteredAttachments := make([]map[string]interface{}, 0) |
|
|
|
|
|
|
|
// Process each attachment
|
|
|
|
for _, attachment := range attachments { |
|
|
|
log.Printf("**** JOB %s: Processing attachment ID: %v", jobID, attachment["id"]) |
|
|
|
|
|
|
|
// Check document types filter
|
|
|
|
if len(docTypes) > 0 { |
|
|
|
typeMatches := false |
|
|
|
|
|
|
|
// Log all attachment details for debugging
|
|
|
|
logAttachmentDetails(jobID, attachment) |
|
|
|
|
|
|
|
// Log docTypes array as it comes from the form
|
|
|
|
log.Printf("**** JOB %s: Doc types from form: %v", jobID, docTypes) |
|
|
|
|
|
|
|
// Get all possible attachment type info
|
|
|
|
var purposeId float64 |
|
|
|
var purpose string |
|
|
|
var typeValue string |
|
|
|
|
|
|
|
// Check all possible type fields
|
|
|
|
if val, ok := attachment["purposeId"].(float64); ok { |
|
|
|
purposeId = val |
|
|
|
log.Printf("**** JOB %s: Found purposeId=%.0f", jobID, purposeId) |
|
|
|
} |
|
|
|
|
|
|
|
if val, ok := attachment["purpose"].(string); ok { |
|
|
|
purpose = val |
|
|
|
log.Printf("**** JOB %s: Found purpose=%s", jobID, purpose) |
|
|
|
} |
|
|
|
|
|
|
|
if val, ok := attachment["type"].(string); ok { |
|
|
|
typeValue = val |
|
|
|
log.Printf("**** JOB %s: Found type=%s", jobID, typeValue) |
|
|
|
} |
|
|
|
|
|
|
|
// Now try to match with each document type from form
|
|
|
|
for _, docType := range docTypes { |
|
|
|
// Clean up the doc type (remove leading zeros)
|
|
|
|
docTypeClean := strings.TrimLeft(docType, "0") |
|
|
|
|
|
|
|
// Try to convert the cleaned doc type to a number
|
|
|
|
if docTypeNum, err := strconv.ParseFloat(docTypeClean, 64); err == nil { |
|
|
|
// Compare with purposeId
|
|
|
|
if purposeId > 0 && purposeId == docTypeNum { |
|
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches purposeId=%.0f", |
|
|
|
jobID, docType, purposeId) |
|
|
|
typeMatches = true |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Try string comparisons if no match yet
|
|
|
|
if !typeMatches && purpose != "" { |
|
|
|
if docType == purpose || docTypeClean == purpose { |
|
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches purpose=%s", |
|
|
|
jobID, docType, purpose) |
|
|
|
typeMatches = true |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if !typeMatches && typeValue != "" { |
|
|
|
if docType == typeValue || docTypeClean == typeValue { |
|
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches type=%s", |
|
|
|
jobID, docType, typeValue) |
|
|
|
typeMatches = true |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if !typeMatches { |
|
|
|
log.Printf("**** JOB %s: No type match found, skipping attachment", jobID) |
|
|
|
continue |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Get document name/filename
|
|
|
|
var filename string |
|
|
|
if nameVal, ok := attachment["fileName"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} else if nameVal, ok := attachment["name"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} else if nameVal, ok := attachment["description"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("**** JOB %s: Attachment filename: %s", jobID, filename) |
|
|
|
|
|
|
|
// Check filename pattern
|
|
|
|
if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) { |
|
|
|
log.Printf("**** JOB %s: Skipping attachment - filename '%s' doesn't match patterns", |
|
|
|
jobID, filename) |
|
|
|
continue |
|
|
|
} |
|
|
|
|
|
|
|
// Check age filter if applicable
|
|
|
|
if ageFilterDays > 0 { |
|
|
|
// Get creation time
|
|
|
|
var createdAt time.Time |
|
|
|
var createdOn string |
|
|
|
var hasDate bool |
|
|
|
|
|
|
|
// Try to get the creation date
|
|
|
|
if created, ok := attachment["createdOn"].(string); ok { |
|
|
|
createdOn = created |
|
|
|
hasDate = true |
|
|
|
} else if created, ok := attachment["created"].(string); ok { |
|
|
|
createdOn = created |
|
|
|
hasDate = true |
|
|
|
} else if lastModified, ok := attachment["lastModified"].(string); ok { |
|
|
|
createdOn = lastModified |
|
|
|
hasDate = true |
|
|
|
} else if createdVal, ok := attachment["created"].(float64); ok { |
|
|
|
createdAt = time.Unix(int64(createdVal), 0) |
|
|
|
createdOn = createdAt.Format(time.RFC3339) |
|
|
|
hasDate = true |
|
|
|
} |
|
|
|
|
|
|
|
if hasDate { |
|
|
|
if parsedTime, err := time.Parse(time.RFC3339, createdOn); err == nil { |
|
|
|
createdAt = parsedTime |
|
|
|
if createdAt.After(cutoffDate) { |
|
|
|
log.Printf("Skipping attachment %s - created on %s is newer than cutoff %s", |
|
|
|
filename, createdAt.Format("2006-01-02"), cutoffDate.Format("2006-01-02")) |
|
|
|
continue // Skip if not old enough
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Use a new variable with a different name for the log message
|
|
|
|
var typeStr string = "unknown" |
|
|
|
if pId, ok := attachment["purposeId"].(float64); ok { |
|
|
|
typeStr = fmt.Sprintf("%.0f", pId) |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("Attachment %s (type: %s) matches all criteria - queued for deletion", |
|
|
|
filename, typeStr) |
|
|
|
|
|
|
|
// If we got here, the attachment passes all filters
|
|
|
|
filteredAttachments = append(filteredAttachments, attachment) |
|
|
|
} |
|
|
|
|
|
|
|
// Update the attachments with the filtered list
|
|
|
|
attachments = filteredAttachments |
|
|
|
log.Printf("**** JOB %s: Final attachments to process after filtering: %d", jobID, len(attachments)) |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("**** JOB %s: Final total attachments: %d", jobID, len(attachments)) |
|
|
|
|
|
|
|
if len(attachments) == 0 { |
|
|
|
log.Printf("**** JOB %s: WARNING! No attachments found after all retrieval attempts") |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Filter attachments based on criteria
|
|
|
|
var attachmentsToDelete []map[string]interface{} |
|
|
|
for _, attachment := range attachments { |
|
|
|
// Get the attachment ID
|
|
|
|
attachmentIDRaw, idOk := attachment["id"].(float64) |
|
|
|
if !idOk { |
|
|
|
log.Printf("Skipping attachment - missing ID field") |
|
|
|
continue |
|
|
|
} |
|
|
|
attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDRaw) |
|
|
|
|
|
|
|
// Get the filename
|
|
|
|
var filename string |
|
|
|
if nameVal, ok := attachment["fileName"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} else if nameVal, ok := attachment["name"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} else if nameVal, ok := attachment["description"].(string); ok { |
|
|
|
filename = nameVal |
|
|
|
} |
|
|
|
|
|
|
|
if filename == "" { |
|
|
|
log.Printf("Attachment %s is missing filename information", attachmentIDStr) |
|
|
|
continue |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("Processing attachment ID: %s, filename: %s", attachmentIDStr, filename) |
|
|
|
|
|
|
|
// Check document type using purposeId which IS available in the data
|
|
|
|
if len(docTypes) > 0 { |
|
|
|
typeMatches := false |
|
|
|
|
|
|
|
// Get purposeId
|
|
|
|
var purposeId float64 |
|
|
|
if val, ok := attachment["purposeId"].(float64); ok { |
|
|
|
purposeId = val |
|
|
|
log.Printf("Attachment %s has purposeId=%.0f", attachmentIDStr, purposeId) |
|
|
|
} else { |
|
|
|
log.Printf("Attachment %s has no purposeId field", attachmentIDStr) |
|
|
|
continue |
|
|
|
} |
|
|
|
|
|
|
|
// Compare with selected document types
|
|
|
|
for _, docType := range docTypes { |
|
|
|
// Form uses "01", "02", etc. but API uses 1, 2, etc. - handle both
|
|
|
|
docTypeClean := strings.TrimLeft(docType, "0") |
|
|
|
if docTypeInt, err := strconv.Atoi(docTypeClean); err == nil { |
|
|
|
if float64(docTypeInt) == purposeId { |
|
|
|
log.Printf("✓ Type match for attachment %s: form value %s matches purposeId %.0f", |
|
|
|
attachmentIDStr, docType, purposeId) |
|
|
|
typeMatches = true |
|
|
|
break |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if !typeMatches { |
|
|
|
// Get purposeId for error message
|
|
|
|
var purposeVal float64 |
|
|
|
if val, ok := attachment["purposeId"].(float64); ok { |
|
|
|
purposeVal = val |
|
|
|
log.Printf("Skipping attachment %s - purposeId %.0f doesn't match any selected types: %v", |
|
|
|
attachmentIDStr, purposeVal, docTypes) |
|
|
|
} else { |
|
|
|
log.Printf("Skipping attachment %s - type doesn't match any selected types: %v", |
|
|
|
attachmentIDStr, docTypes) |
|
|
|
} |
|
|
|
continue |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// Comment out problematic log line
|
|
|
|
// log.Printf("Attachment %s (type: %v, created: %s) matches all criteria - queued for deletion",
|
|
|
|
// filename, purposeId, attachment["createdOn"])
|
|
|
|
|
|
|
|
// Log that we found an attachment to delete
|
|
|
|
log.Printf("Attachment %s matches criteria - will be deleted", filename) |
|
|
|
|
|
|
|
// This attachment matches all criteria
|
|
|
|
attachmentsToDelete = append(attachmentsToDelete, attachment) |
|
|
|
} |
|
|
|
|
|
|
|
jobResult.FilesFound = len(attachmentsToDelete) |
|
|
|
|
|
|
|
// Process deletions with rate limiting
|
|
|
|
var deletionWg sync.WaitGroup |
|
|
|
for _, attachment := range attachmentsToDelete { |
|
|
|
// Use a separate goroutine for each deletion with its own semaphore slot
|
|
|
|
deletionWg.Add(1) |
|
|
|
|
|
|
|
// Important: Create a copy of the attachment for the goroutine to avoid
|
|
|
|
// sharing the loop variable which can cause race conditions
|
|
|
|
attachmentCopy := attachment |
|
|
|
|
|
|
|
go func(att map[string]interface{}) { |
|
|
|
defer deletionWg.Done() |
|
|
|
|
|
|
|
// Acquire a semaphore slot for this deletion operation
|
|
|
|
semaphore <- struct{}{} |
|
|
|
defer func() { <-semaphore }() // Release when done
|
|
|
|
|
|
|
|
attachmentIDFloat := att["id"].(float64) |
|
|
|
attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDFloat) // Convert to string without decimal
|
|
|
|
filename := att["description"].(string) |
|
|
|
|
|
|
|
fileResult := struct { |
|
|
|
Name string |
|
|
|
Success bool |
|
|
|
Error string |
|
|
|
}{ |
|
|
|
Name: filename, |
|
|
|
} |
|
|
|
|
|
|
|
// Delete the attachment
|
|
|
|
err := session.DeleteAttachment(attachmentIDStr) |
|
|
|
|
|
|
|
mu.Lock() |
|
|
|
defer mu.Unlock() |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
fileResult.Success = false |
|
|
|
fileResult.Error = err.Error() |
|
|
|
log.Printf("Error deleting attachment %s: %v", filename, err) |
|
|
|
jobResult.Success = false |
|
|
|
} else { |
|
|
|
fileResult.Success = true |
|
|
|
jobResult.FilesRemoved++ |
|
|
|
log.Printf("Successfully deleted attachment %s", filename) |
|
|
|
} |
|
|
|
|
|
|
|
jobResult.Files = append(jobResult.Files, fileResult) |
|
|
|
|
|
|
|
// Add a slight delay to avoid overwhelming the API
|
|
|
|
time.Sleep(300 * time.Millisecond) |
|
|
|
}(attachmentCopy) |
|
|
|
} |
|
|
|
|
|
|
|
// Wait for all deletions for this job to complete
|
|
|
|
deletionWg.Wait() |
|
|
|
|
|
|
|
mu.Lock() |
|
|
|
results.JobsProcessed++ |
|
|
|
if jobResult.Success { |
|
|
|
results.SuccessCount += jobResult.FilesRemoved |
|
|
|
} else { |
|
|
|
results.ErrorCount += (jobResult.FilesFound - jobResult.FilesRemoved) |
|
|
|
} |
|
|
|
results.TotalFiles += jobResult.FilesFound |
|
|
|
results.JobResults = append(results.JobResults, jobResult) |
|
|
|
mu.Unlock() |
|
|
|
}(jobID) |
|
|
|
} |
|
|
|
|
|
|
|
// Wait for all jobs to complete
|
|
|
|
wg.Wait() |
|
|
|
|
|
|
|
// Generate HTML for results
|
|
|
|
var resultHTML bytes.Buffer |
|
|
|
|
|
|
|
// Add summary section
|
|
|
|
resultHTML.WriteString("<div class=\"upload-summary\">") |
|
|
|
resultHTML.WriteString("<h3>Document Removal Results</h3>") |
|
|
|
resultHTML.WriteString("<div class=\"upload-stats\">") |
|
|
|
|
|
|
|
// Total jobs stat
|
|
|
|
resultHTML.WriteString("<div class=\"stat-box\">") |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.JobsProcessed)) |
|
|
|
resultHTML.WriteString("<div class=\"stat-label\">Total Jobs</div>") |
|
|
|
resultHTML.WriteString("</div>") |
|
|
|
|
|
|
|
// Success stat
|
|
|
|
resultHTML.WriteString("<div class=\"stat-box success-stat\">") |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.SuccessCount)) |
|
|
|
resultHTML.WriteString("<div class=\"stat-label\">Successful Removals</div>") |
|
|
|
resultHTML.WriteString("</div>") |
|
|
|
|
|
|
|
// Failure stat
|
|
|
|
resultHTML.WriteString("<div class=\"stat-box error-stat\">") |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.ErrorCount)) |
|
|
|
resultHTML.WriteString("<div class=\"stat-label\">Failed Removals</div>") |
|
|
|
resultHTML.WriteString("</div>") |
|
|
|
|
|
|
|
// File count stat
|
|
|
|
resultHTML.WriteString("<div class=\"stat-box\">") |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.TotalFiles)) |
|
|
|
resultHTML.WriteString("<div class=\"stat-label\">Files Processed</div>") |
|
|
|
resultHTML.WriteString("</div>") |
|
|
|
|
|
|
|
resultHTML.WriteString("</div>") // End of upload-stats
|
|
|
|
|
|
|
|
// Add completion message
|
|
|
|
if results.ErrorCount == 0 { |
|
|
|
resultHTML.WriteString("<p>All documents were successfully removed from ServiceTrade!</p>") |
|
|
|
} else { |
|
|
|
resultHTML.WriteString("<p>Some documents failed to be removed. See details below.</p>") |
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString("</div>") // End of upload-summary
|
|
|
|
|
|
|
|
// Add detailed job results
|
|
|
|
resultHTML.WriteString("<div class=\"job-results\">") |
|
|
|
|
|
|
|
// Sort job IDs for consistent display
|
|
|
|
sort.Slice(results.JobResults, func(i, j int) bool { |
|
|
|
return results.JobResults[i].JobID < results.JobResults[j].JobID |
|
|
|
}) |
|
|
|
|
|
|
|
for _, jobResult := range results.JobResults { |
|
|
|
// Job result row
|
|
|
|
jobClass := "success" |
|
|
|
if !jobResult.Success { |
|
|
|
jobClass = "error" |
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-result %s\">", jobClass)) |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"job-id\">Job #%s</span>", jobResult.JobID)) |
|
|
|
|
|
|
|
if jobResult.ErrorMsg != "" { |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"error-message\">%s</div>", jobResult.ErrorMsg)) |
|
|
|
} else { |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-summary\">Found %d document(s), removed %d</div>", |
|
|
|
jobResult.FilesFound, jobResult.FilesRemoved)) |
|
|
|
} |
|
|
|
|
|
|
|
// File results
|
|
|
|
if len(jobResult.Files) > 0 { |
|
|
|
resultHTML.WriteString("<div class=\"file-results\">") |
|
|
|
|
|
|
|
for _, file := range jobResult.Files { |
|
|
|
fileClass := "success" |
|
|
|
icon := "✓" |
|
|
|
message := "Successfully removed" |
|
|
|
|
|
|
|
if !file.Success { |
|
|
|
fileClass = "error" |
|
|
|
icon = "✗" |
|
|
|
message = file.Error |
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"file-result %s\">", fileClass)) |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"status-icon\">%s</span>", icon)) |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-name\">%s:</span>", file.Name)) |
|
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-message\">%s</span>", message)) |
|
|
|
resultHTML.WriteString("</div>") |
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString("</div>") // End of file-results
|
|
|
|
} else { |
|
|
|
resultHTML.WriteString("<p>No files processed for this job.</p>") |
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString("</div>") // End of job-result
|
|
|
|
} |
|
|
|
|
|
|
|
resultHTML.WriteString("</div>") // End of job-results
|
|
|
|
|
|
|
|
w.Header().Set("Content-Type", "text/html") |
|
|
|
w.Write(resultHTML.Bytes()) |
|
|
|
} |
|
|
|
|
|
|
|
// Helper function to check if a string is in a slice
|
|
|
|
func stringInSlice(s string, slice []string) bool { |
|
|
|
for _, item := range slice { |
|
|
|
if item == s { |
|
|
|
return true |
|
|
|
} |
|
|
|
} |
|
|
|
return false |
|
|
|
} |
|
|
|
|
|
|
|
// Helper function to check if a string matches any pattern in a slice
|
|
|
|
func matchesAnyPattern(s string, patterns []string) bool { |
|
|
|
for _, pattern := range patterns { |
|
|
|
match, _ := regexp.MatchString("(?i)"+pattern, s) |
|
|
|
if match { |
|
|
|
return true |
|
|
|
} |
|
|
|
} |
|
|
|
return false |
|
|
|
} |
|
|
|
|
|
|
|
// Enhanced debugging function to help understand attachment structure
|
|
|
|
func logAttachmentDetails(jobID string, attachment map[string]interface{}) { |
|
|
|
// Create a detailed view of the attachment
|
|
|
|
attachmentID := "unknown" |
|
|
|
if id, ok := attachment["id"].(float64); ok { |
|
|
|
attachmentID = fmt.Sprintf("%.0f", id) |
|
|
|
} |
|
|
|
|
|
|
|
log.Printf("***** DETAILED ATTACHMENT %s *****", attachmentID) |
|
|
|
for key, value := range attachment { |
|
|
|
log.Printf(" %s = %v (type: %T)", key, value, value) |
|
|
|
} |
|
|
|
log.Printf("***** END ATTACHMENT DETAILS *****") |
|
|
|
} |
|
|
|
|