diff --git a/pkg/models/label.go b/pkg/models/label.go
index d06990bdc8..5658f1069f 100644
--- a/pkg/models/label.go
+++ b/pkg/models/label.go
@@ -151,8 +151,8 @@ func (l *Label) ReadAll(s *xorm.Session, a web.Auth, search string, page int, pe
 		return nil, 0, 0, err
 	}
 
-	return getLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
-		Search:              search,
+	return GetLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
+		Search:              []string{search},
 		User:                u,
 		GetForUser:          u.ID,
 		Page:                page,
diff --git a/pkg/models/label_task.go b/pkg/models/label_task.go
index 0d67c0994f..73eed09b13 100644
--- a/pkg/models/label_task.go
+++ b/pkg/models/label_task.go
@@ -129,9 +129,9 @@ func (lt *LabelTask) ReadAll(s *xorm.Session, a web.Auth, search string, page in
 		return nil, 0, 0, ErrNoRightToSeeTask{lt.TaskID, a.GetID()}
 	}
 
-	return getLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
+	return GetLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
 		User:    &user.User{ID: a.GetID()},
-		Search:  search,
+		Search:  []string{search},
 		Page:    page,
 		TaskIDs: []int64{lt.TaskID},
 	})
@@ -146,7 +146,7 @@ type labelWithTaskID struct {
 // LabelByTaskIDsOptions is a struct to not clutter the function with too many optional parameters.
 type LabelByTaskIDsOptions struct {
 	User                *user.User
-	Search              string
+	Search              []string
 	Page                int
 	PerPage             int
 	TaskIDs             []int64
@@ -155,9 +155,9 @@ type LabelByTaskIDsOptions struct {
 	GetForUser          int64
 }
 
-// Helper function to get all labels for a set of tasks
+// GetLabelsByTaskIDs is a helper function to get all labels for a set of tasks
 // Used when getting all labels for one task as well when getting all lables
-func getLabelsByTaskIDs(s *xorm.Session, opts *LabelByTaskIDsOptions) (ls []*labelWithTaskID, resultCount int, totalEntries int64, err error) {
+func GetLabelsByTaskIDs(s *xorm.Session, opts *LabelByTaskIDsOptions) (ls []*labelWithTaskID, resultCount int, totalEntries int64, err error) {
 	// We still need the task ID when we want to get all labels for a task, but because of this, we get the same label
 	// multiple times when it is associated to more than one task.
 	// Because of this whole thing, we need this extra switch here to only group by Task IDs if needed.
@@ -188,8 +188,14 @@ func getLabelsByTaskIDs(s *xorm.Session, opts *LabelByTaskIDsOptions) (ls []*lab
 	}
 
 	ids := []int64{}
-	if opts.Search != "" {
-		vals := strings.Split(opts.Search, ",")
+
+	for _, search := range opts.Search {
+		search = strings.Trim(search, " ")
+		if search == "" {
+			continue
+		}
+
+		vals := strings.Split(search, ",")
 		for _, val := range vals {
 			v, err := strconv.ParseInt(val, 10, 64)
 			if err != nil {
@@ -203,7 +209,21 @@ func getLabelsByTaskIDs(s *xorm.Session, opts *LabelByTaskIDsOptions) (ls []*lab
 	if len(ids) > 0 {
 		cond = builder.And(cond, builder.In("labels.id", ids))
 	} else {
-		cond = builder.And(cond, db.ILIKE("labels.title", opts.Search))
+
+		if len(opts.Search) > 0 {
+
+			var searchcond builder.Cond
+			for _, search := range opts.Search {
+				search = strings.Trim(search, " ")
+				if search == "" {
+					continue
+				}
+
+				searchcond = builder.Or(searchcond, db.ILIKE("labels.title", search))
+			}
+
+			cond = builder.And(cond, searchcond)
+		}
 	}
 
 	limit, start := getLimitFromPageIndex(opts.Page, opts.PerPage)
@@ -254,7 +274,6 @@ func getLabelsByTaskIDs(s *xorm.Session, opts *LabelByTaskIDsOptions) (ls []*lab
 		Select("count(DISTINCT labels.id)").
 		Join("LEFT", "label_tasks", "label_tasks.label_id = labels.id").
 		Where(cond).
-		And("labels.title LIKE ?", "%"+opts.Search+"%").
 		Count(&Label{})
 	if err != nil {
 		return nil, 0, 0, err
@@ -381,7 +400,7 @@ func (ltb *LabelTaskBulk) Create(s *xorm.Session, a web.Auth) (err error) {
 	if err != nil {
 		return
 	}
-	labels, _, _, err := getLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
+	labels, _, _, err := GetLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
 		TaskIDs: []int64{ltb.TaskID},
 	})
 	if err != nil {
diff --git a/pkg/models/tasks.go b/pkg/models/tasks.go
index 2a66ccbb2c..0c79961cb3 100644
--- a/pkg/models/tasks.go
+++ b/pkg/models/tasks.go
@@ -612,7 +612,7 @@ func addAssigneesToTasks(s *xorm.Session, taskIDs []int64, taskMap map[int64]*Ta
 
 // Get all labels for all the tasks
 func addLabelsToTasks(s *xorm.Session, taskIDs []int64, taskMap map[int64]*Task) (err error) {
-	labels, _, _, err := getLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
+	labels, _, _, err := GetLabelsByTaskIDs(s, &LabelByTaskIDsOptions{
 		TaskIDs: taskIDs,
 		Page:    -1,
 	})
diff --git a/pkg/routes/caldav/listStorageProvider.go b/pkg/routes/caldav/listStorageProvider.go
index 62209634fa..032d3fd691 100644
--- a/pkg/routes/caldav/listStorageProvider.go
+++ b/pkg/routes/caldav/listStorageProvider.go
@@ -388,28 +388,48 @@ func (vcls *VikunjaCaldavListStorage) DeleteResource(rpath string) error {
 }
 
 func persistLabels(s *xorm.Session, a web.Auth, task *models.Task, labels []*models.Label) (err error) {
-	// Find or create Labels by title
+
+	labelTitles := []string{}
+
 	for _, label := range labels {
-		l, err := models.GetLabelSimple(s, &models.Label{Title: label.Title})
+		labelTitles = append(labelTitles, label.Title)
+	}
+
+	u := &user2.User{
+		ID: a.GetID(),
+	}
+
+	// Using readall ensures the current user has the permission to see the labels they provided via caldav.
+	existingLabels, _, _, err := models.GetLabelsByTaskIDs(s, &models.LabelByTaskIDsOptions{
+		Search:              labelTitles,
+		User:                u,
+		GetForUser:          u.ID,
+		GetUnusedLabels:     true,
+		GroupByLabelIDsOnly: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	labelMap := make(map[int64]*models.Label)
+	for _, l := range existingLabels {
+		labelMap[l.ID] = &l.Label
+	}
+
+	for _, label := range labels {
+		if l, has := labelMap[label.ID]; has {
+			label = l
+			continue
+		}
+
+		err = label.Create(s, a)
 		if err != nil {
-			if models.IsErrLabelDoesNotExist(err) {
-				err = label.Create(s, a)
-				if err != nil {
-					return err
-				}
-			} else {
-				return err
-			}
-		} else {
-			*label = *l
+			return err
 		}
 	}
-	// Insert LabelTask relation
-	err = task.UpdateTaskLabels(s, a, labels)
-	if err != nil {
-		return
-	}
-	return nil
+
+	// Create the label <-> task relation
+	return task.UpdateTaskLabels(s, a, labels)
 }
 
 // VikunjaListResourceAdapter holds the actual resource