You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1670 lines
58 KiB
1670 lines
58 KiB
package web
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
root "marmic/servicetrade-toolbox"
|
|
"marmic/servicetrade-toolbox/internal/api"
|
|
"marmic/servicetrade-toolbox/internal/middleware"
|
|
|
|
"github.com/gorilla/mux"
|
|
)
|
|
|
|
// DocumentRemoveHandler handles the document removal page
|
|
func DocumentRemoveHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"Title": "Document Removal",
|
|
"Session": session,
|
|
"DocumentTypes": []map[string]string{
|
|
{"value": "1", "label": "Job Paperwork"},
|
|
{"value": "2", "label": "Job Vendor Bill"},
|
|
{"value": "7", "label": "Generic Attachment"},
|
|
{"value": "10", "label": "Blank Paperwork"},
|
|
{"value": "14", "label": "Job Invoice"},
|
|
},
|
|
}
|
|
|
|
if r.Header.Get("HX-Request") == "true" {
|
|
// For HTMX requests, just send the document_remove partial
|
|
if err := tmpl.ExecuteTemplate(w, "document_remove", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
} else {
|
|
// For full page requests, first render document_remove into a buffer
|
|
var contentBuf bytes.Buffer
|
|
if err := tmpl.ExecuteTemplate(&contentBuf, "document_remove", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
// Add the rendered content to the data for the layout
|
|
data["BodyContent"] = contentBuf.String()
|
|
|
|
// Now render the layout with our content
|
|
if err := tmpl.ExecuteTemplate(w, "layout.html", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// ProcessRemoveCSVHandler processes a CSV file containing job IDs for document removal
|
|
func ProcessRemoveCSVHandler(w http.ResponseWriter, r *http.Request) {
|
|
// We don't use the session in the body but check it for auth
|
|
_, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
// Check if the request method is POST
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
// Parse the multipart form data with a 10MB limit
|
|
if err := r.ParseMultipartForm(10 << 20); err != nil {
|
|
http.Error(w, "Unable to parse form: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get the file from the form
|
|
file, _, err := r.FormFile("csv")
|
|
if err != nil {
|
|
http.Error(w, "Error retrieving file: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
defer file.Close()
|
|
|
|
// Read the CSV data
|
|
csvData, err := csv.NewReader(file).ReadAll()
|
|
if err != nil {
|
|
http.Error(w, "Error reading CSV file: "+err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Extract job IDs from CSV - first column only for simplicity
|
|
var jobIDs []string
|
|
for rowIndex, row := range csvData {
|
|
// Skip header row if present
|
|
if rowIndex == 0 && len(csvData) > 1 {
|
|
// Check if first row looks like a header (non-numeric content)
|
|
_, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
|
if err != nil {
|
|
continue // Skip this row as it's likely a header
|
|
}
|
|
}
|
|
|
|
if len(row) > 0 && row[0] != "" {
|
|
jobID := strings.TrimSpace(row[0])
|
|
if jobID != "" {
|
|
jobIDs = append(jobIDs, jobID)
|
|
}
|
|
}
|
|
}
|
|
|
|
totalJobs := len(jobIDs)
|
|
log.Printf("Extracted %d job IDs from CSV", totalJobs)
|
|
|
|
if totalJobs == 0 {
|
|
http.Error(w, "No valid job IDs found in the CSV file", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Create a hidden input with the job IDs
|
|
jobsValue := strings.Join(jobIDs, ",")
|
|
jobSampleDisplay := getJobSampleDisplay(jobIDs)
|
|
|
|
// Generate HTML for the main response (hidden input for job-ids-removal-container)
|
|
var responseHTML bytes.Buffer
|
|
responseHTML.WriteString(fmt.Sprintf(`<input type="hidden" name="jobIDs" value="%s">`, jobsValue))
|
|
responseHTML.WriteString(fmt.Sprintf(`<p>Found <strong>%d</strong> job(s) in the CSV file</p>`, totalJobs))
|
|
responseHTML.WriteString(fmt.Sprintf(`<div class="csv-sample"><p>Sample job IDs: %s</p></div>`, jobSampleDisplay))
|
|
|
|
// Generate out-of-band swap for the preview section
|
|
responseHTML.WriteString(fmt.Sprintf(`
|
|
<div id="csv-preview-removal" class="fade-me-out" style="display: block !important; margin-top: 1rem;" hx-swap-oob="true">
|
|
<h4>Detected Jobs</h4>
|
|
<div id="csv-preview-content-removal" class="job-list">
|
|
<div class="csv-sample">
|
|
<p>Sample job IDs: %s</p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
`, jobSampleDisplay))
|
|
|
|
// Send the response with the hidden input and preview
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write(responseHTML.Bytes())
|
|
}
|
|
|
|
// After the CSV is processed, a separate request should load job attachments
|
|
// This handler is for Step 2
|
|
func JobSelectionHandler(w http.ResponseWriter, r *http.Request) {
|
|
// We don't use the session directly but check it for auth
|
|
_, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
jobIDs := r.FormValue("jobIDs")
|
|
if jobIDs == "" {
|
|
http.Error(w, "No job IDs provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
jobs := strings.Split(jobIDs, ",")
|
|
if len(jobs) == 0 {
|
|
http.Error(w, "No valid job IDs found", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
var resultHTML bytes.Buffer
|
|
resultHTML.WriteString("<div class=\"job-list-container\">")
|
|
resultHTML.WriteString("<h4>Jobs from CSV</h4>")
|
|
resultHTML.WriteString("<p>Click a job to view and manage its documents.</p>")
|
|
resultHTML.WriteString("<div class=\"job-list\">")
|
|
|
|
for _, jobID := range jobs {
|
|
resultHTML.WriteString(fmt.Sprintf(`
|
|
<div class="job-item">
|
|
<a href="#job-%s" hx-get="/documents/remove/job/%s" hx-target="#job-%s-attachments"
|
|
hx-swap="outerHTML" hx-trigger="click">
|
|
Job ID: %s
|
|
</a>
|
|
<div id="job-%s-attachments"></div>
|
|
</div>
|
|
`, jobID, jobID, jobID, jobID, jobID))
|
|
}
|
|
|
|
resultHTML.WriteString("</div>") // End of job-list
|
|
resultHTML.WriteString("</div>") // End of job-list-container
|
|
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write(resultHTML.Bytes())
|
|
}
|
|
|
|
// GetJobAttachmentsHandler retrieves attachments for a specific job
|
|
func GetJobAttachmentsHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
vars := mux.Vars(r)
|
|
jobID := vars["jobID"]
|
|
renderErrorTemplate(w, "job_attachments", "You must be logged in to use this feature", jobID)
|
|
return
|
|
}
|
|
|
|
vars := mux.Vars(r)
|
|
jobID := vars["jobID"]
|
|
if jobID == "" {
|
|
renderErrorTemplate(w, "job_attachments", "Job ID is required", jobID)
|
|
return
|
|
}
|
|
|
|
// Get attachments for the job
|
|
attachments, err := session.GetJobAttachments(jobID)
|
|
if err != nil {
|
|
renderErrorTemplate(w, "job_attachments", fmt.Sprintf("Failed to get attachments: %v", err), jobID)
|
|
return
|
|
}
|
|
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"JobID": jobID,
|
|
"Attachments": attachments,
|
|
"Session": session,
|
|
}
|
|
|
|
if err := tmpl.ExecuteTemplate(w, "job_attachments", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
// RemoveJobAttachmentsHandler handles the removal of attachments from a job
|
|
func RemoveJobAttachmentsHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
vars := mux.Vars(r)
|
|
jobID := vars["jobID"]
|
|
renderErrorTemplate(w, "job_attachments", "You must be logged in to use this feature", jobID)
|
|
return
|
|
}
|
|
|
|
vars := mux.Vars(r)
|
|
jobID := vars["jobID"]
|
|
if jobID == "" {
|
|
renderErrorTemplate(w, "job_attachments", "Job ID is required", jobID)
|
|
return
|
|
}
|
|
|
|
// Parse the form
|
|
if err := r.ParseForm(); err != nil {
|
|
renderErrorTemplate(w, "job_attachments", fmt.Sprintf("Error parsing form: %v", err), jobID)
|
|
return
|
|
}
|
|
|
|
// Get the attachment IDs to remove
|
|
attachmentIDs := r.PostForm["attachment_ids"]
|
|
if len(attachmentIDs) == 0 {
|
|
renderErrorTemplate(w, "job_attachments", "No attachments selected for deletion", jobID)
|
|
return
|
|
}
|
|
|
|
// Process deletion with rate limiting (max 5 concurrent requests)
|
|
results := struct {
|
|
Success bool
|
|
SuccessCount int
|
|
ErrorCount int
|
|
Files []struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}
|
|
}{
|
|
Success: true,
|
|
Files: make([]struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}, 0, len(attachmentIDs)),
|
|
}
|
|
|
|
// Set up rate limiting
|
|
semaphore := make(chan struct{}, 5) // Allow 5 concurrent requests
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex // Mutex for updating results
|
|
|
|
for _, attachmentID := range attachmentIDs {
|
|
wg.Add(1)
|
|
semaphore <- struct{}{} // Acquire semaphore
|
|
|
|
go func(id string) {
|
|
defer wg.Done()
|
|
defer func() { <-semaphore }() // Release semaphore
|
|
|
|
// Get attachment info first to get the name
|
|
attachmentInfo, err := session.GetAttachmentInfo(id)
|
|
|
|
fileResult := struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}{
|
|
Name: fmt.Sprintf("Attachment ID: %s", id),
|
|
}
|
|
|
|
if err == nil {
|
|
// Get description if available
|
|
if description, ok := attachmentInfo["description"].(string); ok {
|
|
fileResult.Name = description
|
|
}
|
|
}
|
|
|
|
// For all attachment types, we'll use the attachment endpoint for deletion
|
|
// The API endpoint is /attachment/{id} as defined in DeleteAttachment method
|
|
log.Printf("Deleting attachment %s (ID: %s) using attachment endpoint", fileResult.Name, id)
|
|
deleteErr := session.DeleteAttachment(id)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
if deleteErr != nil {
|
|
fileResult.Success = false
|
|
fileResult.Error = deleteErr.Error()
|
|
results.ErrorCount++
|
|
results.Success = false
|
|
} else {
|
|
fileResult.Success = true
|
|
results.SuccessCount++
|
|
}
|
|
|
|
results.Files = append(results.Files, fileResult)
|
|
|
|
// Add a slight delay to avoid overwhelming the API
|
|
time.Sleep(100 * time.Millisecond)
|
|
}(attachmentID)
|
|
}
|
|
|
|
wg.Wait() // Wait for all deletions to complete
|
|
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"JobID": jobID,
|
|
"Session": session,
|
|
"SuccessCount": results.SuccessCount,
|
|
"ErrorCount": results.ErrorCount,
|
|
"JobsProcessed": 1,
|
|
"Results": []map[string]interface{}{
|
|
{
|
|
"JobID": jobID,
|
|
"Success": results.Success,
|
|
"Files": results.Files,
|
|
},
|
|
},
|
|
}
|
|
|
|
// Generate HTML for results that will go to the removal_results div
|
|
if err := tmpl.ExecuteTemplate(w, "removal_results", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
// JobListHandler renders the job list for document removal
|
|
func JobListHandler(w http.ResponseWriter, r *http.Request) {
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
renderErrorTemplate(w, "job_list", "You must be logged in to use this feature")
|
|
return
|
|
}
|
|
|
|
if err := r.ParseForm(); err != nil {
|
|
renderErrorTemplate(w, "job_list", fmt.Sprintf("Error parsing form: %v", err))
|
|
return
|
|
}
|
|
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"JobIDs": r.PostForm["job_ids"],
|
|
"Session": session,
|
|
}
|
|
|
|
if err := tmpl.ExecuteTemplate(w, "job_list", data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Helper function to render error templates
|
|
func renderErrorTemplate(w http.ResponseWriter, templateName, errorMsg string, jobID ...string) {
|
|
tmpl := root.WebTemplates
|
|
data := map[string]interface{}{
|
|
"Error": errorMsg,
|
|
}
|
|
|
|
if len(jobID) > 0 && jobID[0] != "" {
|
|
data["JobID"] = jobID[0]
|
|
}
|
|
|
|
if err := tmpl.ExecuteTemplate(w, templateName, data); err != nil {
|
|
log.Printf("Template execution error: %v", err)
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
}
|
|
}
|
|
|
|
// Helper function to get map keys as a string for logging
|
|
func mapKeysStr(m map[string]interface{}) string {
|
|
if m == nil {
|
|
return "nil"
|
|
}
|
|
keys := make([]string, 0, len(m))
|
|
for k := range m {
|
|
keys = append(keys, k)
|
|
}
|
|
return strings.Join(keys, ", ")
|
|
}
|
|
|
|
// BulkRemoveDocumentsHandler handles bulk removal of documents from multiple jobs
|
|
func BulkRemoveDocumentsHandler(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPost {
|
|
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
|
|
session, ok := r.Context().Value(middleware.SessionKey).(*api.Session)
|
|
if !ok {
|
|
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
|
return
|
|
}
|
|
|
|
// Parse the form
|
|
if err := r.ParseForm(); err != nil {
|
|
http.Error(w, fmt.Sprintf("Error parsing form: %v", err), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get job IDs from the form
|
|
jobIDs := r.FormValue("jobIDs")
|
|
if jobIDs == "" {
|
|
http.Error(w, "No job IDs provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
jobs := strings.Split(jobIDs, ",")
|
|
if len(jobs) == 0 {
|
|
http.Error(w, "No valid job IDs provided", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Get document types to remove (optional)
|
|
var docTypes []string
|
|
if types := r.Form["documentTypes"]; len(types) > 0 {
|
|
docTypes = types
|
|
log.Printf("Filtering by document types: %v", docTypes)
|
|
}
|
|
|
|
// Get filename patterns to match (optional)
|
|
var filenamePatterns []string
|
|
if patterns := r.FormValue("filenamePatterns"); patterns != "" {
|
|
filenamePatterns = strings.Split(patterns, ",")
|
|
for i, p := range filenamePatterns {
|
|
filenamePatterns[i] = strings.TrimSpace(p)
|
|
}
|
|
log.Printf("Filtering by filename patterns: %v", filenamePatterns)
|
|
}
|
|
|
|
// Get age filter (optional)
|
|
var ageFilterDays int
|
|
if ageStr := r.FormValue("age-filter"); ageStr != "" {
|
|
if days, err := strconv.Atoi(ageStr); err == nil && days > 0 {
|
|
ageFilterDays = days
|
|
log.Printf("Using age filter: older than %d days", ageFilterDays)
|
|
}
|
|
}
|
|
|
|
// Calculate cutoff date if using age filter
|
|
var cutoffDate time.Time
|
|
if ageFilterDays > 0 {
|
|
cutoffDate = time.Now().AddDate(0, 0, -ageFilterDays)
|
|
log.Printf("Cutoff date for age filter: %s", cutoffDate.Format("2006-01-02"))
|
|
}
|
|
|
|
// Structure to track results
|
|
type BulkRemovalResult struct {
|
|
JobsProcessed int
|
|
JobsWithErrors int
|
|
TotalFiles int
|
|
SuccessCount int
|
|
ErrorCount int
|
|
JobResults []struct {
|
|
JobID string
|
|
FilesFound int
|
|
FilesRemoved int
|
|
Success bool
|
|
ErrorMsg string
|
|
Files []struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}
|
|
}
|
|
}
|
|
|
|
results := BulkRemovalResult{}
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
semaphore := make(chan struct{}, 5) // Limit concurrent API calls
|
|
|
|
// Process each job
|
|
for _, jobID := range jobs {
|
|
wg.Add(1)
|
|
go func(jobID string) {
|
|
defer wg.Done()
|
|
|
|
log.Printf("Processing job ID: %s for document removal", jobID)
|
|
|
|
jobResult := struct {
|
|
JobID string
|
|
FilesFound int
|
|
FilesRemoved int
|
|
Success bool
|
|
ErrorMsg string
|
|
Files []struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}
|
|
}{
|
|
JobID: jobID,
|
|
Success: true,
|
|
Files: []struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}{},
|
|
}
|
|
|
|
// Check job permissions first
|
|
log.Printf("**** JOB %s: Checking permissions...", jobID)
|
|
hasAccess, reason, err := session.CheckJobPermissions(jobID)
|
|
if err != nil {
|
|
log.Printf("**** JOB %s: Error checking permissions: %v", jobID, err)
|
|
} else {
|
|
log.Printf("**** JOB %s: Permission check result: access=%v, reason=%s",
|
|
jobID, hasAccess, reason)
|
|
}
|
|
|
|
// Create the attachments array that will hold all found documents
|
|
var attachments []map[string]interface{}
|
|
|
|
// Only proceed if we have access or couldn't determine access
|
|
if err != nil || hasAccess {
|
|
// Try job paperwork endpoint
|
|
if stringInSlice("1", docTypes) || len(docTypes) == 0 {
|
|
log.Printf("**** JOB %s: Trying the job paperwork endpoint", jobID)
|
|
paperworkURL := fmt.Sprintf("%s/job/%s/paperwork", api.BaseURL, jobID)
|
|
paperworkReq, err := http.NewRequest("GET", paperworkURL, nil)
|
|
if err == nil {
|
|
paperworkReq.Header.Set("Cookie", session.Cookie)
|
|
paperworkReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending request to job paperwork endpoint: %s", jobID, paperworkURL)
|
|
paperworkResp, err := session.Client.Do(paperworkReq)
|
|
|
|
if err == nil && paperworkResp.StatusCode == http.StatusOK {
|
|
defer paperworkResp.Body.Close()
|
|
paperworkBody, _ := io.ReadAll(paperworkResp.Body)
|
|
|
|
// Log preview of the response
|
|
responsePreview := string(paperworkBody)
|
|
if len(responsePreview) > 300 {
|
|
responsePreview = responsePreview[:300] + "... [truncated]"
|
|
}
|
|
log.Printf("**** JOB %s: Job paperwork response preview: %s", jobID, responsePreview)
|
|
|
|
var paperworkResult map[string]interface{}
|
|
if err := json.Unmarshal(paperworkBody, &paperworkResult); err == nil {
|
|
// Process objects array if it exists
|
|
if objects, ok := paperworkResult["objects"].([]interface{}); ok && len(objects) > 0 {
|
|
log.Printf("**** JOB %s: Found %d paperwork items in objects array", jobID, len(objects))
|
|
|
|
for _, obj := range objects {
|
|
if paperworkMap, ok := obj.(map[string]interface{}); ok {
|
|
// Set purposeId to 1 for job paperwork
|
|
paperworkMap["purposeId"] = float64(1)
|
|
attachments = append(attachments, paperworkMap)
|
|
log.Printf("**** JOB %s: Added job paperwork to attachments", jobID)
|
|
}
|
|
}
|
|
} else if data, ok := paperworkResult["data"].(map[string]interface{}); ok {
|
|
// Check in data for attachments
|
|
if attachmentsArray, ok := data["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 {
|
|
log.Printf("**** JOB %s: Found %d paperwork items in data.attachments", jobID, len(attachmentsArray))
|
|
|
|
for _, att := range attachmentsArray {
|
|
if attMap, ok := att.(map[string]interface{}); ok {
|
|
// Ensure purposeId is set correctly
|
|
attMap["purposeId"] = float64(1)
|
|
attachments = append(attachments, attMap)
|
|
log.Printf("**** JOB %s: Added job paperwork from data.attachments", jobID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Also check other locations in data
|
|
possibleKeys := []string{"paperwork", "objects"}
|
|
for _, key := range possibleKeys {
|
|
if items, ok := data[key].([]interface{}); ok && len(items) > 0 {
|
|
log.Printf("**** JOB %s: Found %d paperwork items in data.%s", jobID, len(items), key)
|
|
|
|
for _, item := range items {
|
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
|
// Set purposeId to 1 for job paperwork
|
|
itemMap["purposeId"] = float64(1)
|
|
attachments = append(attachments, itemMap)
|
|
log.Printf("**** JOB %s: Added job paperwork from data.%s", jobID, key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: Job paperwork endpoint failed or returned non-200 status: %v", jobID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Try job invoice endpoint
|
|
if stringInSlice("14", docTypes) || len(docTypes) == 0 {
|
|
log.Printf("**** JOB %s: Trying the job invoice endpoint", jobID)
|
|
invoiceURL := fmt.Sprintf("%s/job/%s/invoice", api.BaseURL, jobID)
|
|
invoiceReq, err := http.NewRequest("GET", invoiceURL, nil)
|
|
if err == nil {
|
|
invoiceReq.Header.Set("Cookie", session.Cookie)
|
|
invoiceReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending request to job invoice endpoint: %s", jobID, invoiceURL)
|
|
invoiceResp, err := session.Client.Do(invoiceReq)
|
|
|
|
if err == nil && invoiceResp.StatusCode == http.StatusOK {
|
|
defer invoiceResp.Body.Close()
|
|
invoiceBody, _ := io.ReadAll(invoiceResp.Body)
|
|
|
|
// Log preview of the response
|
|
responsePreview := string(invoiceBody)
|
|
if len(responsePreview) > 300 {
|
|
responsePreview = responsePreview[:300] + "... [truncated]"
|
|
}
|
|
log.Printf("**** JOB %s: Job invoice response preview: %s", jobID, responsePreview)
|
|
|
|
var invoiceResult map[string]interface{}
|
|
if err := json.Unmarshal(invoiceBody, &invoiceResult); err == nil {
|
|
// Process objects array if it exists
|
|
if objects, ok := invoiceResult["objects"].([]interface{}); ok && len(objects) > 0 {
|
|
log.Printf("**** JOB %s: Found %d job invoices in objects array", jobID, len(objects))
|
|
|
|
for _, obj := range objects {
|
|
if invoiceMap, ok := obj.(map[string]interface{}); ok {
|
|
// Set purposeId to 14 for job invoices
|
|
invoiceMap["purposeId"] = float64(14)
|
|
attachments = append(attachments, invoiceMap)
|
|
log.Printf("**** JOB %s: Added job invoice to attachments", jobID)
|
|
}
|
|
}
|
|
} else if data, ok := invoiceResult["data"].(map[string]interface{}); ok {
|
|
// Check in data for attachments
|
|
possibleKeys := []string{"invoices", "attachments", "objects"}
|
|
for _, key := range possibleKeys {
|
|
if items, ok := data[key].([]interface{}); ok && len(items) > 0 {
|
|
log.Printf("**** JOB %s: Found %d invoices in data.%s", jobID, len(items), key)
|
|
|
|
for _, item := range items {
|
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
|
// Set purposeId to 14 for job invoices
|
|
itemMap["purposeId"] = float64(14)
|
|
attachments = append(attachments, itemMap)
|
|
log.Printf("**** JOB %s: Added job invoice from data.%s", jobID, key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: Job invoice endpoint failed or returned non-200 status: %v", jobID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Try generic attachment endpoint
|
|
if stringInSlice("7", docTypes) || len(docTypes) == 0 {
|
|
log.Printf("**** JOB %s: Trying the generic attachment endpoint", jobID)
|
|
genericURL := fmt.Sprintf("%s/job/%s/attachment", api.BaseURL, jobID)
|
|
genericReq, err := http.NewRequest("GET", genericURL, nil)
|
|
if err == nil {
|
|
genericReq.Header.Set("Cookie", session.Cookie)
|
|
genericReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending request to generic attachment endpoint: %s", jobID, genericURL)
|
|
genericResp, err := session.Client.Do(genericReq)
|
|
|
|
if err == nil && genericResp.StatusCode == http.StatusOK {
|
|
defer genericResp.Body.Close()
|
|
genericBody, _ := io.ReadAll(genericResp.Body)
|
|
|
|
// Log preview of the response
|
|
responsePreview := string(genericBody)
|
|
if len(responsePreview) > 300 {
|
|
responsePreview = responsePreview[:300] + "... [truncated]"
|
|
}
|
|
log.Printf("**** JOB %s: Generic attachment response preview: %s", jobID, responsePreview)
|
|
|
|
var genericResult map[string]interface{}
|
|
if err := json.Unmarshal(genericBody, &genericResult); err == nil {
|
|
// Process objects array if it exists
|
|
if objects, ok := genericResult["objects"].([]interface{}); ok && len(objects) > 0 {
|
|
log.Printf("**** JOB %s: Found %d generic attachments in objects array", jobID, len(objects))
|
|
|
|
for _, obj := range objects {
|
|
if attachMap, ok := obj.(map[string]interface{}); ok {
|
|
// Set purposeId to 7 for generic attachments
|
|
attachMap["purposeId"] = float64(7)
|
|
attachments = append(attachments, attachMap)
|
|
log.Printf("**** JOB %s: Added generic attachment to attachments", jobID)
|
|
}
|
|
}
|
|
} else if data, ok := genericResult["data"].(map[string]interface{}); ok {
|
|
// Check in data for attachments
|
|
possibleKeys := []string{"attachments", "objects"}
|
|
for _, key := range possibleKeys {
|
|
if items, ok := data[key].([]interface{}); ok && len(items) > 0 {
|
|
log.Printf("**** JOB %s: Found %d generic attachments in data.%s", jobID, len(items), key)
|
|
|
|
for _, item := range items {
|
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
|
// Set purposeId to 7 for generic attachments
|
|
itemMap["purposeId"] = float64(7)
|
|
attachments = append(attachments, itemMap)
|
|
log.Printf("**** JOB %s: Added generic attachment from data.%s", jobID, key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: Generic attachment endpoint failed or returned non-200 status: %v", jobID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Try vendor bill endpoint
|
|
if stringInSlice("2", docTypes) || len(docTypes) == 0 {
|
|
log.Printf("**** JOB %s: Trying the vendor invoice endpoint", jobID)
|
|
vendorInvoiceURL := fmt.Sprintf("%s/job/%s/vendorinvoice", api.BaseURL, jobID)
|
|
vendorInvoiceReq, err := http.NewRequest("GET", vendorInvoiceURL, nil)
|
|
if err == nil {
|
|
vendorInvoiceReq.Header.Set("Cookie", session.Cookie)
|
|
vendorInvoiceReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending request to vendor invoice endpoint: %s", jobID, vendorInvoiceURL)
|
|
vendorInvoiceResp, err := session.Client.Do(vendorInvoiceReq)
|
|
|
|
if err == nil && vendorInvoiceResp.StatusCode == http.StatusOK {
|
|
defer vendorInvoiceResp.Body.Close()
|
|
vendorInvoiceBody, _ := io.ReadAll(vendorInvoiceResp.Body)
|
|
|
|
// Log preview of the response
|
|
responsePreview := string(vendorInvoiceBody)
|
|
if len(responsePreview) > 300 {
|
|
responsePreview = responsePreview[:300] + "... [truncated]"
|
|
}
|
|
log.Printf("**** JOB %s: Vendor invoice response preview: %s", jobID, responsePreview)
|
|
|
|
var vendorInvoiceResult map[string]interface{}
|
|
if err := json.Unmarshal(vendorInvoiceBody, &vendorInvoiceResult); err == nil {
|
|
// Process objects array if it exists
|
|
if objects, ok := vendorInvoiceResult["objects"].([]interface{}); ok && len(objects) > 0 {
|
|
log.Printf("**** JOB %s: Found %d vendor invoices in objects array", jobID, len(objects))
|
|
|
|
for _, obj := range objects {
|
|
if invoiceMap, ok := obj.(map[string]interface{}); ok {
|
|
// Set purposeId to 2 for vendor bills
|
|
invoiceMap["purposeId"] = float64(2)
|
|
attachments = append(attachments, invoiceMap)
|
|
log.Printf("**** JOB %s: Added vendor invoice to attachments", jobID)
|
|
}
|
|
}
|
|
} else if data, ok := vendorInvoiceResult["data"].(map[string]interface{}); ok {
|
|
// Check in data for attachments
|
|
possibleKeys := []string{"invoices", "vendorInvoices", "attachments", "objects"}
|
|
for _, key := range possibleKeys {
|
|
if items, ok := data[key].([]interface{}); ok && len(items) > 0 {
|
|
log.Printf("**** JOB %s: Found %d vendor bills in data.%s", jobID, len(items), key)
|
|
|
|
for _, item := range items {
|
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
|
// Set purposeId to 2 for vendor bills
|
|
itemMap["purposeId"] = float64(2)
|
|
attachments = append(attachments, itemMap)
|
|
log.Printf("**** JOB %s: Added vendor bill from data.%s", jobID, key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: Vendor invoice endpoint failed or returned non-200 status: %v", jobID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Then continue with general paperwork endpoint to catch any we might have missed
|
|
log.Printf("**** JOB %s: Trying general paperwork endpoint", jobID)
|
|
|
|
// Directly try to get paperwork using the specialized API
|
|
log.Printf("**** JOB %s: Using specialized paperwork API", jobID)
|
|
paperworkItems, err := session.GetJobPaperwork(jobID)
|
|
|
|
if err != nil {
|
|
log.Printf("**** JOB %s: Error getting paperwork: %v", jobID, err)
|
|
} else if len(paperworkItems) > 0 {
|
|
log.Printf("**** JOB %s: GetJobPaperwork returned %d paperwork items",
|
|
jobID, len(paperworkItems))
|
|
|
|
// Add all paperwork items to attachments
|
|
for _, item := range paperworkItems {
|
|
log.Printf("**** JOB %s: Adding item from GetJobPaperwork: %v",
|
|
jobID, mapKeysStr(item))
|
|
attachments = append(attachments, item)
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: No paperwork found using specialized API", jobID)
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: WARNING: No access to this job - reason: %s", jobID, reason)
|
|
mu.Lock()
|
|
jobResult.Success = false
|
|
jobResult.ErrorMsg = fmt.Sprintf("Cannot access job: %s", reason)
|
|
results.JobResults = append(results.JobResults, jobResult)
|
|
results.JobsWithErrors++
|
|
mu.Unlock()
|
|
return
|
|
}
|
|
|
|
// Try alternate method (always try both methods to see what data is available)
|
|
log.Printf("**** JOB %s: Retrieving attachments using GetAttachmentsForJob", jobID)
|
|
apiResponse, err := session.GetAttachmentsForJob(jobID)
|
|
if err != nil {
|
|
log.Printf("**** JOB %s: Error in GetAttachmentsForJob: %v", jobID, err)
|
|
} else {
|
|
// Log the structure of the response to understand format
|
|
rootKeys := make([]string, 0)
|
|
for k := range apiResponse {
|
|
rootKeys = append(rootKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: GetAttachmentsForJob returned response with root keys: %s",
|
|
jobID, strings.Join(rootKeys, ", "))
|
|
|
|
// Check if we have a data object
|
|
if data, ok := apiResponse["data"].(map[string]interface{}); ok {
|
|
dataKeys := make([]string, 0)
|
|
for k := range data {
|
|
dataKeys = append(dataKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: data object keys: %s",
|
|
jobID, strings.Join(dataKeys, ", "))
|
|
}
|
|
|
|
// Check if we have paperwork_data
|
|
if paperworkData, ok := apiResponse["paperwork_data"].(map[string]interface{}); ok {
|
|
dataKeys := make([]string, 0)
|
|
for k := range paperworkData {
|
|
dataKeys = append(dataKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: paperwork_data keys: %s",
|
|
jobID, strings.Join(dataKeys, ", "))
|
|
|
|
// Check if paperwork_data.data.attachments exists
|
|
if paperworkDataInner, ok := paperworkData["data"].(map[string]interface{}); ok {
|
|
if attachmentsArray, ok := paperworkDataInner["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 {
|
|
log.Printf("**** JOB %s: Found %d attachments in paperwork_data.data.attachments",
|
|
jobID, len(attachmentsArray))
|
|
|
|
// Process each attachment and add to our collection
|
|
for _, attachment := range attachmentsArray {
|
|
if attachmentMap, ok := attachment.(map[string]interface{}); ok {
|
|
attachments = append(attachments, attachmentMap)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
log.Printf("**** JOB %s: Total attachments gathered: %d", jobID, len(attachments))
|
|
|
|
if len(attachments) == 0 {
|
|
log.Printf("**** JOB %s: No attachments found yet, trying direct paperwork endpoint", jobID)
|
|
|
|
// Directly try to get vendor bills using a specific endpoint
|
|
log.Printf("**** JOB %s: Trying vendor bill specific endpoint", jobID)
|
|
vendorBillURL := fmt.Sprintf("%s/job/%s/vendor-bill", api.BaseURL, jobID)
|
|
vendorBillReq, err := http.NewRequest("GET", vendorBillURL, nil)
|
|
if err == nil {
|
|
vendorBillReq.Header.Set("Cookie", session.Cookie)
|
|
vendorBillReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending request to vendor bill endpoint: %s", jobID, vendorBillURL)
|
|
vendorBillResp, err := session.Client.Do(vendorBillReq)
|
|
|
|
if err == nil && vendorBillResp.StatusCode == http.StatusOK {
|
|
defer vendorBillResp.Body.Close()
|
|
vendorBillBody, _ := io.ReadAll(vendorBillResp.Body)
|
|
|
|
// Log full response structure for debugging
|
|
log.Printf("**** JOB %s: Full vendor bill response: %s", jobID, string(vendorBillBody))
|
|
|
|
var vendorBillResult map[string]interface{}
|
|
if err := json.Unmarshal(vendorBillBody, &vendorBillResult); err == nil {
|
|
// Log all root keys in the response
|
|
rootKeys := make([]string, 0)
|
|
for k := range vendorBillResult {
|
|
rootKeys = append(rootKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: Vendor bill response root keys: %s",
|
|
jobID, strings.Join(rootKeys, ", "))
|
|
|
|
// Check if data exists and log all its keys
|
|
if data, ok := vendorBillResult["data"].(map[string]interface{}); ok {
|
|
dataKeys := make([]string, 0)
|
|
for k := range data {
|
|
dataKeys = append(dataKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: Vendor bill data keys: %s",
|
|
jobID, strings.Join(dataKeys, ", "))
|
|
|
|
// First try vendorBills directly
|
|
if vendorBills, ok := data["vendorBills"].([]interface{}); ok && len(vendorBills) > 0 {
|
|
log.Printf("**** JOB %s: Found %d vendor bills in data.vendorBills", jobID, len(vendorBills))
|
|
|
|
for _, bill := range vendorBills {
|
|
if billMap, ok := bill.(map[string]interface{}); ok {
|
|
// Set purposeId to 2 for vendor bills
|
|
billMap["purposeId"] = float64(2)
|
|
attachments = append(attachments, billMap)
|
|
}
|
|
}
|
|
} else {
|
|
// Try other possible locations
|
|
log.Printf("**** JOB %s: No vendorBills found in data, checking other locations", jobID)
|
|
|
|
// Try each possible location for the vendor bills
|
|
possibleKeys := []string{"objects", "attachments", "bills", "paperwork", "documents"}
|
|
for _, key := range possibleKeys {
|
|
if items, ok := data[key].([]interface{}); ok && len(items) > 0 {
|
|
log.Printf("**** JOB %s: Found %d items in data.%s", jobID, len(items), key)
|
|
|
|
// Log the structure of the first item
|
|
if itemMap, ok := items[0].(map[string]interface{}); ok {
|
|
itemKeys := make([]string, 0)
|
|
for k := range itemMap {
|
|
itemKeys = append(itemKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: First item in data.%s has keys: %s",
|
|
jobID, key, strings.Join(itemKeys, ", "))
|
|
|
|
// Log the first item as JSON for inspection
|
|
if itemJSON, err := json.Marshal(itemMap); err == nil {
|
|
log.Printf("**** JOB %s: First item in data.%s: %s",
|
|
jobID, key, string(itemJSON))
|
|
}
|
|
}
|
|
|
|
// Add all items as attachments
|
|
for _, item := range items {
|
|
if itemMap, ok := item.(map[string]interface{}); ok {
|
|
// Set purposeId to 2 for vendor bills
|
|
itemMap["purposeId"] = float64(2)
|
|
attachments = append(attachments, itemMap)
|
|
log.Printf("**** JOB %s: Added vendor bill from data.%s", jobID, key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// If data is not a map, check for top-level objects
|
|
log.Printf("**** JOB %s: No data object in vendor bill response or it's not a map", jobID)
|
|
|
|
if objects, ok := vendorBillResult["objects"].([]interface{}); ok && len(objects) > 0 {
|
|
log.Printf("**** JOB %s: Found %d objects at root level", jobID, len(objects))
|
|
|
|
for _, obj := range objects {
|
|
if objMap, ok := obj.(map[string]interface{}); ok {
|
|
// Set purposeId to 2 for vendor bills
|
|
objMap["purposeId"] = float64(2)
|
|
attachments = append(attachments, objMap)
|
|
log.Printf("**** JOB %s: Added vendor bill from root.objects", jobID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.Printf("**** JOB %s: Vendor bill endpoint failed or returned non-200 status: %v", jobID, err)
|
|
}
|
|
}
|
|
|
|
// Also try direct paperwork endpoint
|
|
log.Printf("**** JOB %s: Trying direct paperwork endpoint", jobID)
|
|
paperworkURL := fmt.Sprintf("%s/job/%s/paperwork", api.BaseURL, jobID)
|
|
paperworkReq, err := http.NewRequest("GET", paperworkURL, nil)
|
|
if err == nil {
|
|
paperworkReq.Header.Set("Cookie", session.Cookie)
|
|
paperworkReq.Header.Set("Accept", "application/json")
|
|
|
|
log.Printf("**** JOB %s: Sending direct request to %s", jobID, paperworkURL)
|
|
paperworkResp, err := session.Client.Do(paperworkReq)
|
|
|
|
if err == nil && paperworkResp.StatusCode == http.StatusOK {
|
|
defer paperworkResp.Body.Close()
|
|
paperworkBody, _ := io.ReadAll(paperworkResp.Body)
|
|
|
|
// Log preview of the response
|
|
responsePreview := string(paperworkBody)
|
|
if len(responsePreview) > 200 {
|
|
responsePreview = responsePreview[:200] + "... [truncated]"
|
|
}
|
|
log.Printf("**** JOB %s: Direct paperwork response preview: %s", jobID, responsePreview)
|
|
|
|
// Parse the response
|
|
var paperworkResult map[string]interface{}
|
|
if err := json.Unmarshal(paperworkBody, &paperworkResult); err == nil {
|
|
// Log the structure of the response
|
|
rootKeys := make([]string, 0)
|
|
for k := range paperworkResult {
|
|
rootKeys = append(rootKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: Direct paperwork response keys: %s",
|
|
jobID, strings.Join(rootKeys, ", "))
|
|
|
|
// Check for data.attachments
|
|
if data, ok := paperworkResult["data"].(map[string]interface{}); ok {
|
|
dataKeys := make([]string, 0)
|
|
for k := range data {
|
|
dataKeys = append(dataKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: Direct paperwork data keys: %s",
|
|
jobID, strings.Join(dataKeys, ", "))
|
|
|
|
// Extract attachments from data.attachments
|
|
if attachmentsArray, ok := data["attachments"].([]interface{}); ok && len(attachmentsArray) > 0 {
|
|
log.Printf("**** JOB %s: Found %d attachments in direct paperwork response",
|
|
jobID, len(attachmentsArray))
|
|
|
|
// Loop through the attachments and add to our collection
|
|
for i, attachment := range attachmentsArray {
|
|
if attachmentMap, ok := attachment.(map[string]interface{}); ok {
|
|
// Log details of the first attachment to understand the structure
|
|
if i == 0 {
|
|
log.Printf("**** JOB %s: First attachment structure: %+v", jobID, attachmentMap)
|
|
attKeys := make([]string, 0)
|
|
for k := range attachmentMap {
|
|
attKeys = append(attKeys, k)
|
|
}
|
|
log.Printf("**** JOB %s: First attachment keys: %s", jobID, strings.Join(attKeys, ", "))
|
|
}
|
|
|
|
// Add to our attachments collection
|
|
attachments = append(attachments, attachmentMap)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log attachment count after direct endpoint
|
|
log.Printf("**** JOB %s: Attachments found after direct paperwork call: %d", jobID, len(attachments))
|
|
|
|
// Deduplicate attachments to avoid processing the same ones multiple times
|
|
originalCount := len(attachments)
|
|
attachments = deduplicateAttachments(attachments)
|
|
if len(attachments) < originalCount {
|
|
log.Printf("**** JOB %s: Removed %d duplicate attachments, %d unique attachments remain",
|
|
jobID, originalCount-len(attachments), len(attachments))
|
|
}
|
|
|
|
// Now actually apply the filters
|
|
filteredAttachments := make([]map[string]interface{}, 0)
|
|
|
|
// Process each attachment
|
|
for _, attachment := range attachments {
|
|
log.Printf("**** JOB %s: Processing attachment ID: %v", jobID, attachment["id"])
|
|
|
|
// Check document types filter
|
|
if len(docTypes) > 0 {
|
|
typeMatches := false
|
|
|
|
// Log all attachment details for debugging
|
|
logAttachmentDetails(jobID, attachment)
|
|
|
|
// Log docTypes array as it comes from the form
|
|
log.Printf("**** JOB %s: Doc types from form: %v", jobID, docTypes)
|
|
|
|
// Get all possible attachment type info
|
|
var purposeId float64
|
|
var purpose string
|
|
var typeValue string
|
|
|
|
// Check all possible type fields
|
|
if val, ok := attachment["purposeId"].(float64); ok {
|
|
purposeId = val
|
|
log.Printf("**** JOB %s: Found purposeId=%.0f", jobID, purposeId)
|
|
}
|
|
|
|
if val, ok := attachment["purpose"].(string); ok {
|
|
purpose = val
|
|
log.Printf("**** JOB %s: Found purpose=%s", jobID, purpose)
|
|
}
|
|
|
|
if val, ok := attachment["type"].(string); ok {
|
|
typeValue = val
|
|
log.Printf("**** JOB %s: Found type=%s", jobID, typeValue)
|
|
}
|
|
|
|
// Now try to match with each document type from form
|
|
for _, docType := range docTypes {
|
|
// Clean up the doc type (remove leading zeros)
|
|
docTypeClean := strings.TrimLeft(docType, "0")
|
|
|
|
// Try to convert the cleaned doc type to a number
|
|
if docTypeNum, err := strconv.ParseFloat(docTypeClean, 64); err == nil {
|
|
// Compare with purposeId
|
|
if purposeId > 0 && purposeId == docTypeNum {
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches purposeId=%.0f",
|
|
jobID, docType, purposeId)
|
|
typeMatches = true
|
|
break
|
|
}
|
|
}
|
|
|
|
// Try string comparisons if no match yet
|
|
if !typeMatches && purpose != "" {
|
|
if docType == purpose || docTypeClean == purpose {
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches purpose=%s",
|
|
jobID, docType, purpose)
|
|
typeMatches = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if !typeMatches && typeValue != "" {
|
|
if docType == typeValue || docTypeClean == typeValue {
|
|
log.Printf("**** JOB %s: MATCH! docType=%s matches type=%s",
|
|
jobID, docType, typeValue)
|
|
typeMatches = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if !typeMatches {
|
|
log.Printf("**** JOB %s: No type match found, skipping attachment", jobID)
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Get filename from any available field, don't assume description exists
|
|
var filename string
|
|
if desc, ok := attachment["description"].(string); ok && desc != "" {
|
|
filename = desc
|
|
} else if name, ok := attachment["name"].(string); ok && name != "" {
|
|
filename = name
|
|
} else if fname, ok := attachment["fileName"].(string); ok && fname != "" {
|
|
filename = fname
|
|
} else {
|
|
// If no name is available, use the ID as the name
|
|
filename = fmt.Sprintf("Attachment ID: %s", attachment["id"])
|
|
}
|
|
|
|
log.Printf("**** JOB %s: Attachment filename: %s", jobID, filename)
|
|
|
|
// Check filename pattern
|
|
if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) {
|
|
log.Printf("**** JOB %s: Skipping attachment - filename '%s' doesn't match patterns",
|
|
jobID, filename)
|
|
continue
|
|
}
|
|
|
|
// Check age filter if applicable
|
|
if ageFilterDays > 0 {
|
|
// Get creation time
|
|
var createdAt time.Time
|
|
var createdOn string
|
|
var hasDate bool
|
|
|
|
// Try to get the creation date
|
|
if created, ok := attachment["createdOn"].(string); ok {
|
|
createdOn = created
|
|
hasDate = true
|
|
} else if created, ok := attachment["created"].(string); ok {
|
|
createdOn = created
|
|
hasDate = true
|
|
} else if lastModified, ok := attachment["lastModified"].(string); ok {
|
|
createdOn = lastModified
|
|
hasDate = true
|
|
} else if createdVal, ok := attachment["created"].(float64); ok {
|
|
createdAt = time.Unix(int64(createdVal), 0)
|
|
createdOn = createdAt.Format(time.RFC3339)
|
|
hasDate = true
|
|
}
|
|
|
|
if hasDate {
|
|
if parsedTime, err := time.Parse(time.RFC3339, createdOn); err == nil {
|
|
createdAt = parsedTime
|
|
if createdAt.After(cutoffDate) {
|
|
log.Printf("Skipping attachment %s - created on %s is newer than cutoff %s",
|
|
filename, createdAt.Format("2006-01-02"), cutoffDate.Format("2006-01-02"))
|
|
continue // Skip if not old enough
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Use a new variable with a different name for the log message
|
|
var typeStr string = "unknown"
|
|
if pId, ok := attachment["purposeId"].(float64); ok {
|
|
typeStr = fmt.Sprintf("%.0f", pId)
|
|
}
|
|
|
|
log.Printf("Attachment %s (type: %s) matches all criteria - queued for deletion",
|
|
filename, typeStr)
|
|
|
|
// If we got here, the attachment passes all filters
|
|
filteredAttachments = append(filteredAttachments, attachment)
|
|
}
|
|
|
|
// Update the attachments with the filtered list
|
|
attachments = filteredAttachments
|
|
log.Printf("**** JOB %s: Final attachments to process after filtering: %d", jobID, len(attachments))
|
|
}
|
|
|
|
log.Printf("**** JOB %s: Final total attachments: %d", jobID, len(attachments))
|
|
|
|
if len(attachments) == 0 {
|
|
log.Printf("**** JOB %s: WARNING! No attachments found after all retrieval attempts", jobID)
|
|
} else {
|
|
// Deduplicate again before continuing with deletion, in case multiple methods found the same attachments
|
|
originalCount := len(attachments)
|
|
attachments = deduplicateAttachments(attachments)
|
|
if len(attachments) < originalCount {
|
|
log.Printf("**** JOB %s: Final deduplication removed %d duplicates, %d unique attachments remain",
|
|
jobID, originalCount-len(attachments), len(attachments))
|
|
}
|
|
}
|
|
}
|
|
|
|
// Filter attachments based on criteria
|
|
var attachmentsToDelete []map[string]interface{}
|
|
for _, attachment := range attachments {
|
|
// Get the attachment ID
|
|
attachmentIDRaw, idOk := attachment["id"].(float64)
|
|
if !idOk {
|
|
log.Printf("Skipping attachment - missing ID field")
|
|
continue
|
|
}
|
|
attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDRaw)
|
|
|
|
// Safely get filename from any available field
|
|
var filename string
|
|
if desc, ok := attachment["description"].(string); ok && desc != "" {
|
|
filename = desc
|
|
} else if name, ok := attachment["name"].(string); ok && name != "" {
|
|
filename = name
|
|
} else if fname, ok := attachment["fileName"].(string); ok && fname != "" {
|
|
filename = fname
|
|
} else {
|
|
// If no name is available, use the ID as the name
|
|
filename = fmt.Sprintf("Attachment ID: %s", attachmentIDStr)
|
|
}
|
|
|
|
if filename == "" {
|
|
log.Printf("Attachment %s is missing filename information", attachmentIDStr)
|
|
continue
|
|
}
|
|
|
|
log.Printf("Processing attachment ID: %s, filename: %s", attachmentIDStr, filename)
|
|
|
|
// Check document type using purposeId which IS available in the data
|
|
if len(docTypes) > 0 {
|
|
typeMatches := false
|
|
|
|
// Get purposeId
|
|
var purposeId float64
|
|
if val, ok := attachment["purposeId"].(float64); ok {
|
|
purposeId = val
|
|
log.Printf("Attachment %s has purposeId=%.0f", attachmentIDStr, purposeId)
|
|
} else {
|
|
log.Printf("Attachment %s has no purposeId field", attachmentIDStr)
|
|
continue
|
|
}
|
|
|
|
// Compare with selected document types
|
|
for _, docType := range docTypes {
|
|
// Form uses "01", "02", etc. but API uses 1, 2, etc. - handle both
|
|
docTypeClean := strings.TrimLeft(docType, "0")
|
|
if docTypeInt, err := strconv.Atoi(docTypeClean); err == nil {
|
|
if float64(docTypeInt) == purposeId {
|
|
log.Printf("✓ Type match for attachment %s: form value %s matches purposeId %.0f",
|
|
attachmentIDStr, docType, purposeId)
|
|
typeMatches = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if !typeMatches {
|
|
// Get purposeId for error message
|
|
var purposeVal float64
|
|
if val, ok := attachment["purposeId"].(float64); ok {
|
|
purposeVal = val
|
|
log.Printf("Skipping attachment %s - purposeId %.0f doesn't match any selected types: %v",
|
|
attachmentIDStr, purposeVal, docTypes)
|
|
} else {
|
|
log.Printf("Skipping attachment %s - type doesn't match any selected types: %v",
|
|
attachmentIDStr, docTypes)
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Check filename pattern
|
|
if len(filenamePatterns) > 0 && !matchesAnyPattern(filename, filenamePatterns) {
|
|
log.Printf("Skipping attachment %s - filename '%s' doesn't match patterns: %v",
|
|
attachmentIDStr, filename, filenamePatterns)
|
|
continue
|
|
}
|
|
|
|
// Check age filter if applicable
|
|
if ageFilterDays > 0 {
|
|
// Get creation time
|
|
var createdAt time.Time
|
|
var createdOn string
|
|
var hasDate bool
|
|
|
|
// Try to get the creation date
|
|
if created, ok := attachment["createdOn"].(string); ok {
|
|
createdOn = created
|
|
hasDate = true
|
|
} else if created, ok := attachment["created"].(string); ok {
|
|
createdOn = created
|
|
hasDate = true
|
|
} else if lastModified, ok := attachment["lastModified"].(string); ok {
|
|
createdOn = lastModified
|
|
hasDate = true
|
|
} else if createdVal, ok := attachment["created"].(float64); ok {
|
|
createdAt = time.Unix(int64(createdVal), 0)
|
|
createdOn = createdAt.Format(time.RFC3339)
|
|
hasDate = true
|
|
}
|
|
|
|
if hasDate {
|
|
if parsedTime, err := time.Parse(time.RFC3339, createdOn); err == nil {
|
|
createdAt = parsedTime
|
|
if createdAt.After(cutoffDate) {
|
|
log.Printf("Skipping attachment %s - created on %s is newer than cutoff %s",
|
|
filename, createdAt.Format("2006-01-02"), cutoffDate.Format("2006-01-02"))
|
|
continue // Skip if not old enough
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Log that we found an attachment to delete
|
|
log.Printf("Attachment %s matches criteria - will be deleted", filename)
|
|
|
|
// This attachment matches all criteria
|
|
attachmentsToDelete = append(attachmentsToDelete, attachment)
|
|
}
|
|
|
|
jobResult.FilesFound = len(attachmentsToDelete)
|
|
|
|
// Process deletions with rate limiting
|
|
var deletionWg sync.WaitGroup
|
|
for _, attachment := range attachmentsToDelete {
|
|
// Use a separate goroutine for each deletion with its own semaphore slot
|
|
deletionWg.Add(1)
|
|
|
|
// Important: Create a copy of the attachment for the goroutine to avoid
|
|
// sharing the loop variable which can cause race conditions
|
|
attachmentCopy := attachment
|
|
|
|
go func(att map[string]interface{}) {
|
|
defer deletionWg.Done()
|
|
|
|
// Acquire a semaphore slot for this deletion operation
|
|
semaphore <- struct{}{}
|
|
defer func() { <-semaphore }() // Release when done
|
|
|
|
attachmentIDFloat := att["id"].(float64)
|
|
attachmentIDStr := fmt.Sprintf("%.0f", attachmentIDFloat) // Convert to string without decimal
|
|
|
|
// Safely get filename from any available field
|
|
var filename string
|
|
if desc, ok := att["description"].(string); ok && desc != "" {
|
|
filename = desc
|
|
} else if name, ok := att["name"].(string); ok && name != "" {
|
|
filename = name
|
|
} else if fname, ok := att["fileName"].(string); ok && fname != "" {
|
|
filename = fname
|
|
} else {
|
|
// If no name is available, use the ID as the name
|
|
filename = fmt.Sprintf("Attachment ID: %s", attachmentIDStr)
|
|
}
|
|
|
|
fileResult := struct {
|
|
Name string
|
|
Success bool
|
|
Error string
|
|
}{
|
|
Name: filename,
|
|
}
|
|
|
|
// For all attachment types, we'll use the attachment endpoint for deletion
|
|
// The API endpoint is /attachment/{id} as defined in DeleteAttachment method
|
|
log.Printf("Deleting attachment %s (ID: %s) using attachment endpoint", filename, attachmentIDStr)
|
|
deleteErr := session.DeleteAttachment(attachmentIDStr)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
|
|
if deleteErr != nil {
|
|
fileResult.Success = false
|
|
fileResult.Error = deleteErr.Error()
|
|
log.Printf("Error deleting attachment %s: %v", filename, deleteErr)
|
|
jobResult.Success = false
|
|
} else {
|
|
fileResult.Success = true
|
|
jobResult.FilesRemoved++
|
|
log.Printf("Successfully deleted attachment %s", filename)
|
|
}
|
|
|
|
jobResult.Files = append(jobResult.Files, fileResult)
|
|
|
|
// Add a slight delay to avoid overwhelming the API
|
|
time.Sleep(300 * time.Millisecond)
|
|
}(attachmentCopy)
|
|
}
|
|
|
|
// Wait for all deletions for this job to complete
|
|
deletionWg.Wait()
|
|
|
|
mu.Lock()
|
|
results.JobsProcessed++
|
|
if jobResult.Success {
|
|
results.SuccessCount += jobResult.FilesRemoved
|
|
} else {
|
|
results.ErrorCount += (jobResult.FilesFound - jobResult.FilesRemoved)
|
|
}
|
|
results.TotalFiles += jobResult.FilesFound
|
|
results.JobResults = append(results.JobResults, jobResult)
|
|
mu.Unlock()
|
|
}(jobID)
|
|
}
|
|
|
|
// Wait for all jobs to complete
|
|
wg.Wait()
|
|
|
|
// Generate HTML for results
|
|
var resultHTML bytes.Buffer
|
|
|
|
// Add summary section
|
|
resultHTML.WriteString("<div class=\"upload-summary\">")
|
|
resultHTML.WriteString("<h3>Document Removal Results</h3>")
|
|
resultHTML.WriteString("<div class=\"upload-stats\">")
|
|
|
|
// Total jobs stat
|
|
resultHTML.WriteString("<div class=\"stat-box\">")
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.JobsProcessed))
|
|
resultHTML.WriteString("<div class=\"stat-label\">Total Jobs</div>")
|
|
resultHTML.WriteString("</div>")
|
|
|
|
// Success stat
|
|
resultHTML.WriteString("<div class=\"stat-box success-stat\">")
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.SuccessCount))
|
|
resultHTML.WriteString("<div class=\"stat-label\">Successful Removals</div>")
|
|
resultHTML.WriteString("</div>")
|
|
|
|
// Failure stat
|
|
resultHTML.WriteString("<div class=\"stat-box error-stat\">")
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.ErrorCount))
|
|
resultHTML.WriteString("<div class=\"stat-label\">Failed Removals</div>")
|
|
resultHTML.WriteString("</div>")
|
|
|
|
// File count stat
|
|
resultHTML.WriteString("<div class=\"stat-box\">")
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"stat-value\">%d</div>", results.TotalFiles))
|
|
resultHTML.WriteString("<div class=\"stat-label\">Files Processed</div>")
|
|
resultHTML.WriteString("</div>")
|
|
|
|
resultHTML.WriteString("</div>") // End of upload-stats
|
|
|
|
// Add completion message
|
|
if results.ErrorCount == 0 {
|
|
resultHTML.WriteString("<p>All documents were successfully removed from ServiceTrade!</p>")
|
|
} else {
|
|
resultHTML.WriteString("<p>Some documents failed to be removed. See details below.</p>")
|
|
}
|
|
|
|
resultHTML.WriteString("</div>") // End of upload-summary
|
|
|
|
// Add detailed job results
|
|
resultHTML.WriteString("<div class=\"job-results\">")
|
|
|
|
// Sort job IDs for consistent display
|
|
sort.Slice(results.JobResults, func(i, j int) bool {
|
|
return results.JobResults[i].JobID < results.JobResults[j].JobID
|
|
})
|
|
|
|
for _, jobResult := range results.JobResults {
|
|
// Job result row
|
|
jobClass := "success"
|
|
if !jobResult.Success {
|
|
jobClass = "error"
|
|
}
|
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-result %s\">", jobClass))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"job-id\">Job ID: %s</span>", jobResult.JobID))
|
|
|
|
if jobResult.ErrorMsg != "" {
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"error-message\">%s</div>", jobResult.ErrorMsg))
|
|
} else {
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"job-summary\">Found %d document(s), removed %d</div>",
|
|
jobResult.FilesFound, jobResult.FilesRemoved))
|
|
}
|
|
|
|
// File results
|
|
if len(jobResult.Files) > 0 {
|
|
resultHTML.WriteString("<div class=\"file-results\">")
|
|
|
|
for _, file := range jobResult.Files {
|
|
fileClass := "success"
|
|
icon := "✓"
|
|
message := "Successfully removed"
|
|
|
|
if !file.Success {
|
|
fileClass = "error"
|
|
icon = "✗"
|
|
message = file.Error
|
|
}
|
|
|
|
resultHTML.WriteString(fmt.Sprintf("<div class=\"file-result %s\">", fileClass))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"status-icon\">%s</span>", icon))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-name\">%s:</span>", file.Name))
|
|
resultHTML.WriteString(fmt.Sprintf("<span class=\"file-message\">%s</span>", message))
|
|
resultHTML.WriteString("</div>")
|
|
}
|
|
|
|
resultHTML.WriteString("</div>") // End of file-results
|
|
} else {
|
|
resultHTML.WriteString("<p>No files processed for this job.</p>")
|
|
}
|
|
|
|
resultHTML.WriteString("</div>") // End of job-result
|
|
}
|
|
|
|
resultHTML.WriteString("</div>") // End of job-results
|
|
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write(resultHTML.Bytes())
|
|
}
|
|
|
|
// Helper function to check if a string is in a slice
|
|
func stringInSlice(s string, slice []string) bool {
|
|
for _, item := range slice {
|
|
if item == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Helper function to check if a string matches any pattern in a slice
|
|
func matchesAnyPattern(s string, patterns []string) bool {
|
|
// Convert the string to lowercase for case-insensitive comparison
|
|
sLower := strings.ToLower(s)
|
|
|
|
for _, pattern := range patterns {
|
|
// Check if the pattern is a wildcard pattern (contains *)
|
|
if strings.Contains(pattern, "*") {
|
|
// Convert wildcard pattern to regex
|
|
regexPattern := strings.ReplaceAll(pattern, "*", ".*")
|
|
match, _ := regexp.MatchString("(?i)^"+regexPattern+"$", s)
|
|
if match {
|
|
return true
|
|
}
|
|
} else {
|
|
// For non-wildcard patterns, check for exact match (case-insensitive)
|
|
if sLower == strings.ToLower(pattern) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Enhanced debugging function to help understand attachment structure
|
|
func logAttachmentDetails(jobID string, attachment map[string]interface{}) {
|
|
// Create a detailed view of the attachment
|
|
attachmentID := "unknown"
|
|
if id, ok := attachment["id"].(float64); ok {
|
|
attachmentID = fmt.Sprintf("%.0f", id)
|
|
}
|
|
|
|
log.Printf("***** JOB %s - DETAILED ATTACHMENT %s *****", jobID, attachmentID)
|
|
for key, value := range attachment {
|
|
log.Printf(" JOB %s: %s = %v (type: %T)", jobID, key, value, value)
|
|
}
|
|
log.Printf("***** JOB %s - END ATTACHMENT DETAILS *****", jobID)
|
|
}
|
|
|
|
// Helper function to deduplicate attachments based on ID
|
|
func deduplicateAttachments(attachments []map[string]interface{}) []map[string]interface{} {
|
|
seen := make(map[string]bool)
|
|
uniqueAttachments := make([]map[string]interface{}, 0)
|
|
|
|
for _, attachment := range attachments {
|
|
// Get the ID as a string for deduplication
|
|
var idStr string
|
|
if id, ok := attachment["id"].(float64); ok {
|
|
idStr = fmt.Sprintf("%.0f", id)
|
|
} else if id, ok := attachment["id"].(string); ok {
|
|
idStr = id
|
|
} else {
|
|
// If no valid ID, just add it (should not happen)
|
|
uniqueAttachments = append(uniqueAttachments, attachment)
|
|
continue
|
|
}
|
|
|
|
// Only add if we haven't seen this ID before
|
|
if !seen[idStr] {
|
|
seen[idStr] = true
|
|
uniqueAttachments = append(uniqueAttachments, attachment)
|
|
}
|
|
}
|
|
|
|
return uniqueAttachments
|
|
}
|
|
|