From 43f24661d77d08489313f2f375b623bc2ad84492 Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 15 Mar 2024 16:09:09 +0100 Subject: [PATCH] feat(views): save view and position in Typesense --- pkg/models/listeners.go | 11 +++++ pkg/models/task_collection_sort.go | 41 ++++++++-------- pkg/models/task_search.go | 51 +++++++++++--------- pkg/models/tasks.go | 1 - pkg/models/typesense.go | 76 ++++++++++++++++++++---------- 5 files changed, 111 insertions(+), 69 deletions(-) diff --git a/pkg/models/listeners.go b/pkg/models/listeners.go index 9809d77c8..e8d87c877 100644 --- a/pkg/models/listeners.go +++ b/pkg/models/listeners.go @@ -19,6 +19,8 @@ package models import ( "context" "encoding/json" + "github.com/typesense/typesense-go/typesense/api" + "github.com/typesense/typesense-go/typesense/api/pointer" "strconv" "time" @@ -534,6 +536,15 @@ func (l *AddTaskToTypesense) Handle(msg *message.Message) (err error) { return err } + _, err = typesenseClient.Collection("tasks"). + Documents(). + Delete(context.Background(), &api.DeleteDocumentsParams{ + FilterBy: pointer.String("task_id:" + strconv.FormatInt(event.Task.ID, 10)), + }) + if err != nil { + return err + } + _, err = typesenseClient.Collection("tasks"). Documents(). Create(context.Background(), ttask) diff --git a/pkg/models/task_collection_sort.go b/pkg/models/task_collection_sort.go index fb8b797f2..bd686e049 100644 --- a/pkg/models/task_collection_sort.go +++ b/pkg/models/task_collection_sort.go @@ -27,26 +27,27 @@ type ( ) const ( - taskPropertyID string = "id" - taskPropertyTitle string = "title" - taskPropertyDescription string = "description" - taskPropertyDone string = "done" - taskPropertyDoneAt string = "done_at" - taskPropertyDueDate string = "due_date" - taskPropertyCreatedByID string = "created_by_id" - taskPropertyProjectID string = "project_id" - taskPropertyRepeatAfter string = "repeat_after" - taskPropertyPriority string = "priority" - taskPropertyStartDate string = "start_date" - taskPropertyEndDate string = "end_date" - taskPropertyHexColor string = "hex_color" - taskPropertyPercentDone string = "percent_done" - taskPropertyUID string = "uid" - taskPropertyCreated string = "created" - taskPropertyUpdated string = "updated" - taskPropertyPosition string = "position" - taskPropertyBucketID string = "bucket_id" - taskPropertyIndex string = "index" + taskPropertyID string = "id" + taskPropertyTitle string = "title" + taskPropertyDescription string = "description" + taskPropertyDone string = "done" + taskPropertyDoneAt string = "done_at" + taskPropertyDueDate string = "due_date" + taskPropertyCreatedByID string = "created_by_id" + taskPropertyProjectID string = "project_id" + taskPropertyRepeatAfter string = "repeat_after" + taskPropertyPriority string = "priority" + taskPropertyStartDate string = "start_date" + taskPropertyEndDate string = "end_date" + taskPropertyHexColor string = "hex_color" + taskPropertyPercentDone string = "percent_done" + taskPropertyUID string = "uid" + taskPropertyCreated string = "created" + taskPropertyUpdated string = "updated" + taskPropertyPosition string = "position" + taskPropertyBucketID string = "bucket_id" + taskPropertyIndex string = "index" + taskPropertyProjectViewID string = "project_view_id" ) const ( diff --git a/pkg/models/task_search.go b/pkg/models/task_search.go index 6b0b4ad2f..3283c37c6 100644 --- a/pkg/models/task_search.go +++ b/pkg/models/task_search.go @@ -416,29 +416,6 @@ func convertParsedFilterToTypesense(rawFilters []*taskFilter) (filterBy string, func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, totalCount int64, err error) { - var sortbyFields []string - for i, param := range opts.sortby { - // Validate the params - if err := param.validate(); err != nil { - return nil, totalCount, err - } - - // Typesense does not allow sorting by ID, so we sort by created timestamp instead - if param.sortBy == "id" { - param.sortBy = "created" - } - - sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) - - if i == 2 { - // Typesense supports up to 3 sorting parameters - // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters - break - } - } - - sortby := strings.Join(sortbyFields, ",") - projectIDStrings := []string{} for _, id := range opts.projectIDs { projectIDStrings = append(projectIDStrings, strconv.FormatInt(id, 10)) @@ -454,6 +431,34 @@ func (t *typesenseTaskSearcher) Search(opts *taskSearchOptions) (tasks []*Task, "(" + filter + ")", } + var sortbyFields []string + for i, param := range opts.sortby { + // Validate the params + if err := param.validate(); err != nil { + return nil, totalCount, err + } + + // Typesense does not allow sorting by ID, so we sort by created timestamp instead + if param.sortBy == taskPropertyID { + param.sortBy = taskPropertyCreated + } + + if param.sortBy == taskPropertyPosition { + filterBy = append(filterBy, "project_view_id: "+strconv.FormatInt(param.projectViewID, 10)) + break + } + + sortbyFields = append(sortbyFields, param.sortBy+"(missing_values:last):"+param.orderBy.String()) + + if i == 2 { + // Typesense supports up to 3 sorting parameters + // https://typesense.org/docs/0.25.0/api/search.html#ranking-and-sorting-parameters + break + } + } + + sortby := strings.Join(sortbyFields, ",") + //////////////// // Actual search diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go index 1e325cd06..074086e64 100644 --- a/pkg/models/tasks.go +++ b/pkg/models/tasks.go @@ -258,7 +258,6 @@ func getTaskIndexFromSearchString(s string) (index int64) { return } -//nolint:gocyclo func getRawTasksForProjects(s *xorm.Session, projects []*Project, a web.Auth, opts *taskSearchOptions) (tasks []*Task, resultCount int, totalItems int64, err error) { // If the user does not have any projects, don't try to get any tasks diff --git a/pkg/models/typesense.go b/pkg/models/typesense.go index 512ee8349..92df10126 100644 --- a/pkg/models/typesense.go +++ b/pkg/models/typesense.go @@ -19,6 +19,8 @@ package models import ( "context" "fmt" + "strconv" + "strings" "time" "code.vikunja.io/api/pkg/config" @@ -157,6 +159,10 @@ func CreateTypesenseCollections() error { Name: "created_by_id", Type: "int64", }, + { + Name: "project_view_id", + Type: "int64", + }, { Name: "reminders", Type: "object[]", // TODO @@ -243,14 +249,17 @@ func ReindexAllTasks() (err error) { return } -func getTypesenseTaskForTask(s *xorm.Session, task *Task, projectsCache map[int64]*Project) (ttask *typesenseTask, err error) { +func getTypesenseTaskForTask(s *xorm.Session, task *Task, projectsCache map[int64]*Project) (ttasks []*typesenseTask, err error) { positions := []*TaskPosition{} err = s.Where("task_id = ?", task.ID).Find(&positions) if err != nil { return } - ttask = convertTaskToTypesenseTask(task, positions) + for _, position := range positions { + ttask := convertTaskToTypesenseTask(task, position) + ttasks = append(ttasks, ttask) + } var p *Project if projectsCache == nil { @@ -271,11 +280,15 @@ func getTypesenseTaskForTask(s *xorm.Session, task *Task, projectsCache map[int6 } comment := &TaskComment{TaskID: task.ID} - ttask.Comments, _, _, err = comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1) + comments, _, _, err := comment.ReadAll(s, &user.User{ID: p.OwnerID}, "", -1, -1) if err != nil { return nil, fmt.Errorf("could not fetch comments for task %d: %s", task.ID, err.Error()) } + for _, t := range ttasks { + t.Comments = comments + } + return } @@ -292,16 +305,31 @@ func reindexTasksInTypesense(s *xorm.Session, tasks map[int64]*Task) (err error) } projects := make(map[int64]*Project) - typesenseTasks := []interface{}{} + taskIDs := []string{} + for _, task := range tasks { - ttask, err := getTypesenseTaskForTask(s, task, projects) + ttasks, err := getTypesenseTaskForTask(s, task, projects) if err != nil { return err } - typesenseTasks = append(typesenseTasks, ttask) + for _, ttask := range ttasks { + typesenseTasks = append(typesenseTasks, ttask) + } + + taskIDs = append(taskIDs, strconv.FormatInt(task.ID, 10)) + } + + _, err = typesenseClient.Collection("tasks"). + Documents(). + Delete(context.Background(), &api.DeleteDocumentsParams{ + FilterBy: pointer.String("task_id:[" + strings.Join(taskIDs, ",") + "]"), + }) + if err != nil { + log.Errorf("Could not delete old tasks in Typesense", err) + return err } _, err = typesenseClient.Collection("tasks"). @@ -398,6 +426,7 @@ func indexDummyTask() (err error) { type typesenseTask struct { ID string `json:"id"` + TaskID string `json:"task_id"` Title string `json:"title"` Description string `json:"description"` Done bool `json:"done"` @@ -422,17 +451,22 @@ type typesenseTask struct { Assignees interface{} `json:"assignees"` Labels interface{} `json:"labels"` //RelatedTasks interface{} `json:"related_tasks"` // TODO - Attachments interface{} `json:"attachments"` - Comments interface{} `json:"comments"` - Positions []*struct { - Position float64 `json:"position"` - ProjectViewID int64 `json:"project_view_id"` - } `json:"positions"` + Attachments interface{} `json:"attachments"` + Comments interface{} `json:"comments"` + Position float64 `json:"position"` + ProjectViewID int64 `json:"project_view_id"` } -func convertTaskToTypesenseTask(task *Task, positions []*TaskPosition) *typesenseTask { +func convertTaskToTypesenseTask(task *Task, position *TaskPosition) *typesenseTask { + + var projectViewID int64 + if position != nil { + projectViewID = position.ProjectViewID + } + tt := &typesenseTask{ - ID: fmt.Sprintf("%d", task.ID), + ID: fmt.Sprintf("%d_%d", task.ID, projectViewID), + TaskID: fmt.Sprintf("%d", task.ID), Title: task.Title, Description: task.Description, Done: task.Done, @@ -457,7 +491,9 @@ func convertTaskToTypesenseTask(task *Task, positions []*TaskPosition) *typesens Assignees: task.Assignees, Labels: task.Labels, //RelatedTasks: task.RelatedTasks, - Attachments: task.Attachments, + Attachments: task.Attachments, + Position: position.Position, + ProjectViewID: projectViewID, } if task.DoneAt.IsZero() { @@ -473,16 +509,6 @@ func convertTaskToTypesenseTask(task *Task, positions []*TaskPosition) *typesens tt.EndDate = nil } - for _, position := range positions { - tt.Positions = append(tt.Positions, &struct { - Position float64 `json:"position"` - ProjectViewID int64 `json:"project_view_id"` - }{ - Position: position.Position, - ProjectViewID: position.ProjectViewID, - }) - } - return tt }