mirror of
https://github.com/abhinavxd/libredesk.git
synced 2025-11-05 06:23:27 +00:00
- Implemented MarkdownToHTML function using goldmark for converting markdown content to HTML. - Added CleanJSONResponse function to remove markdown code blocks from LLM responses. - Updated stringutil tests to remove unnecessary test cases for empty strings and special characters. refactor: Update SQL schema for knowledge base and help center - Introduced ai_knowledge_type enum for knowledge base categorization. - Added help_center_id reference in inboxes table. - Enhanced help_centers table with default_locale column. - Changed data types from INTEGER to INT for consistency across tables. - Renamed ai_custom_answers table to ai_knowledge_base and adjusted its schema. fix: Remove unnecessary CSS filter from default icon in widget - Cleaned up widget.js by removing the brightness filter from the default icon styling.
349 lines
8.9 KiB
Go
349 lines
8.9 KiB
Go
// Package stringutil provides string utility functions.
|
|
package stringutil
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/mail"
|
|
"net/url"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/k3a/html2text"
|
|
"github.com/yuin/goldmark"
|
|
"github.com/yuin/goldmark/extension"
|
|
"github.com/yuin/goldmark/renderer/html"
|
|
)
|
|
|
|
const (
|
|
PasswordDummy = "•"
|
|
)
|
|
|
|
var (
|
|
regexpNonAlNum = regexp.MustCompile(`[^a-zA-Z0-9\-_\.]+`)
|
|
regexpSpaces = regexp.MustCompile(`[\s]+`)
|
|
)
|
|
|
|
// HTML2Text converts HTML to text.
|
|
func HTML2Text(html string) string {
|
|
return strings.TrimSpace(html2text.HTML2Text(html))
|
|
}
|
|
|
|
// SanitizeFilename sanitizes the provided filename.
|
|
func SanitizeFilename(fName string) string {
|
|
// Trim whitespace.
|
|
name := strings.TrimSpace(fName)
|
|
|
|
// Replace whitespace and "/" with "-"
|
|
name = regexpSpaces.ReplaceAllString(name, "-")
|
|
|
|
// Remove or replace any non-alphanumeric characters
|
|
name = regexpNonAlNum.ReplaceAllString(name, "")
|
|
|
|
// Convert to lowercase
|
|
name = strings.ToLower(name)
|
|
return filepath.Base(name)
|
|
}
|
|
|
|
// GenerateSlug generates a URL-friendly slug from a title or name.
|
|
func GenerateSlug(title string, prefixRandom bool) string {
|
|
// Trim whitespace
|
|
slug := strings.TrimSpace(title)
|
|
|
|
// Convert to lowercase
|
|
slug = strings.ToLower(slug)
|
|
|
|
// Replace spaces and special characters with hyphens
|
|
slug = regexpSpaces.ReplaceAllString(slug, "-")
|
|
|
|
// Remove any non-alphanumeric characters except hyphens and underscores
|
|
slugRegex := regexp.MustCompile(`[^a-z0-9\-_]+`)
|
|
slug = slugRegex.ReplaceAllString(slug, "")
|
|
|
|
// Remove multiple consecutive hyphens
|
|
multiHyphens := regexp.MustCompile(`-+`)
|
|
slug = multiHyphens.ReplaceAllString(slug, "-")
|
|
|
|
// Trim leading/trailing hyphens
|
|
slug = strings.Trim(slug, "-")
|
|
|
|
// If slug is empty after processing, generate a random string as slug
|
|
if slug == "" {
|
|
randomSlug, err := RandomAlphanumeric(12)
|
|
if err != nil {
|
|
slug = "untitled"
|
|
} else {
|
|
slug = randomSlug
|
|
}
|
|
}
|
|
|
|
if prefixRandom {
|
|
// Generate a random alphanumeric prefix
|
|
randomPrefix, err := RandomAlphanumeric(12)
|
|
if err != nil {
|
|
return "untitled"
|
|
}
|
|
// Prepend the random prefix to the slug
|
|
slug = fmt.Sprintf("%s-%s", randomPrefix, slug)
|
|
}
|
|
|
|
return slug
|
|
}
|
|
|
|
// RandomAlphanumeric generates a random alphanumeric string of length n.
|
|
func RandomAlphanumeric(n int) (string, error) {
|
|
const dictionary = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
|
|
|
bytes := make([]byte, n)
|
|
|
|
if _, err := rand.Read(bytes); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
for k, v := range bytes {
|
|
bytes[k] = dictionary[v%byte(len(dictionary))]
|
|
}
|
|
|
|
return string(bytes), nil
|
|
}
|
|
|
|
// RandomNumeric generates a random numeric string of length n.
|
|
func RandomNumeric(n int) (string, error) {
|
|
const dictionary = "0123456789"
|
|
|
|
bytes := make([]byte, n)
|
|
|
|
if _, err := rand.Read(bytes); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
for k, v := range bytes {
|
|
bytes[k] = dictionary[v%byte(len(dictionary))]
|
|
}
|
|
|
|
return string(bytes), nil
|
|
}
|
|
|
|
// GetPathFromURL extracts the path from a URL.
|
|
func GetPathFromURL(u string) (string, error) {
|
|
parsedURL, err := url.Parse(u)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return parsedURL.Path, nil
|
|
}
|
|
|
|
// RemoveEmpty removes empty strings from a slice of strings.
|
|
func RemoveEmpty(s []string) []string {
|
|
var r []string
|
|
for _, str := range s {
|
|
if str != "" {
|
|
r = append(r, str)
|
|
}
|
|
}
|
|
return r
|
|
}
|
|
|
|
// GenerateEmailMessageID generates a RFC-compliant Message-ID for an email, does not include the angle brackets.
|
|
// The client is expected to wrap the returned string in angle brackets.
|
|
func GenerateEmailMessageID(messageID string, fromAddress string) (string, error) {
|
|
if messageID == "" {
|
|
return "", fmt.Errorf("messageID cannot be empty")
|
|
}
|
|
|
|
// Parse from address
|
|
addr, err := mail.ParseAddress(fromAddress)
|
|
if err != nil {
|
|
return "", fmt.Errorf("invalid from address: %w", err)
|
|
}
|
|
|
|
// Extract domain with validation
|
|
parts := strings.Split(addr.Address, "@")
|
|
if len(parts) != 2 || parts[1] == "" {
|
|
return "", fmt.Errorf("invalid domain in from address")
|
|
}
|
|
domain := parts[1]
|
|
|
|
// Generate cryptographic random component
|
|
random := make([]byte, 8)
|
|
if _, err := rand.Read(random); err != nil {
|
|
return "", fmt.Errorf("failed to generate random bytes: %w", err)
|
|
}
|
|
|
|
// Sanitize messageID for email Message-ID
|
|
cleaner := regexp.MustCompile(`[^\w.-]`) // Allow only alphanum, ., -, _
|
|
cleanmessageID := cleaner.ReplaceAllString(messageID, "_")
|
|
|
|
// Ensure cleaned messageID isn't empty
|
|
if cleanmessageID == "" {
|
|
return "", fmt.Errorf("messageID became empty after sanitization")
|
|
}
|
|
|
|
// Build RFC-compliant Message-ID
|
|
return fmt.Sprintf("%s-%d-%s@%s",
|
|
cleanmessageID,
|
|
time.Now().UnixNano(), // Nanosecond precision
|
|
strings.TrimRight(base64.URLEncoding.EncodeToString(random), "="), // URL-safe base64 without padding
|
|
domain,
|
|
), nil
|
|
}
|
|
|
|
// ReverseSlice reverses a slice of strings in place.
|
|
func ReverseSlice(source []string) {
|
|
for i, j := 0, len(source)-1; i < j; i, j = i+1, j-1 {
|
|
source[i], source[j] = source[j], source[i]
|
|
}
|
|
}
|
|
|
|
// RemoveItemByValue removes all instances of a value from a slice of strings.
|
|
func RemoveItemByValue(slice []string, value string) []string {
|
|
result := []string{}
|
|
for _, v := range slice {
|
|
if v != value {
|
|
result = append(result, v)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// FormatDuration formats a duration as a string.
|
|
func FormatDuration(d time.Duration, includeSeconds bool) string {
|
|
d = d.Round(time.Second)
|
|
h := int64(d.Hours())
|
|
d -= time.Duration(h) * time.Hour
|
|
m := int64(d.Minutes())
|
|
d -= time.Duration(m) * time.Minute
|
|
s := int64(d.Seconds())
|
|
|
|
var parts []string
|
|
if h > 0 {
|
|
parts = append(parts, fmt.Sprintf("%d hours", h))
|
|
}
|
|
if m >= 0 {
|
|
parts = append(parts, fmt.Sprintf("%d minutes", m))
|
|
}
|
|
if s > 0 && includeSeconds {
|
|
parts = append(parts, fmt.Sprintf("%d seconds", s))
|
|
}
|
|
return strings.Join(parts, " ")
|
|
}
|
|
|
|
// ValidEmail returns true if it's a valid email else return false.
|
|
func ValidEmail(email string) bool {
|
|
addr, err := mail.ParseAddress(email)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return addr.Name == "" && addr.Address == email
|
|
}
|
|
|
|
// ExtractEmail extracts the email address from a string.
|
|
func ExtractEmail(s string) (string, error) {
|
|
addr, err := mail.ParseAddress(s)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return addr.Address, nil
|
|
}
|
|
|
|
// DedupAndExcludeString returns a deduplicated []string excluding empty and a specific value.
|
|
func DedupAndExcludeString(list []string, exclude string) []string {
|
|
seen := make(map[string]struct{}, len(list))
|
|
cleaned := make([]string, 0, len(list))
|
|
for _, s := range list {
|
|
if s == "" || s == exclude {
|
|
continue
|
|
}
|
|
if _, ok := seen[s]; !ok {
|
|
seen[s] = struct{}{}
|
|
cleaned = append(cleaned, s)
|
|
}
|
|
}
|
|
return cleaned
|
|
}
|
|
|
|
// ComputeRecipients computes new recipients using last message's recipients and direction.
|
|
func ComputeRecipients(
|
|
from, to, cc, bcc []string,
|
|
contactEmail, inboxEmail string,
|
|
lastMessageIncoming bool,
|
|
) (finalTo, finalCC, finalBCC []string) {
|
|
if lastMessageIncoming {
|
|
if len(from) > 0 {
|
|
finalTo = from
|
|
} else if contactEmail != "" {
|
|
finalTo = []string{contactEmail}
|
|
}
|
|
} else {
|
|
if len(to) > 0 {
|
|
finalTo = to
|
|
} else if contactEmail != "" {
|
|
finalTo = []string{contactEmail}
|
|
}
|
|
}
|
|
|
|
finalCC = append([]string{}, cc...)
|
|
|
|
if lastMessageIncoming {
|
|
if len(to) > 0 {
|
|
finalCC = append(finalCC, to...)
|
|
}
|
|
if contactEmail != "" && !slices.Contains(finalTo, contactEmail) && !slices.Contains(finalCC, contactEmail) {
|
|
finalCC = append(finalCC, contactEmail)
|
|
}
|
|
}
|
|
|
|
finalTo = DedupAndExcludeString(finalTo, inboxEmail)
|
|
finalCC = DedupAndExcludeString(finalCC, inboxEmail)
|
|
// BCC is one-time only, user is supposed to add it manually.
|
|
finalBCC = []string{}
|
|
|
|
return
|
|
}
|
|
|
|
// MarkdownToHTML converts markdown content to HTML using goldmark.
|
|
func MarkdownToHTML(markdown string) (string, error) {
|
|
// Create goldmark parser with safe configuration
|
|
md := goldmark.New(
|
|
goldmark.WithExtensions(extension.GFM),
|
|
goldmark.WithRendererOptions(
|
|
html.WithHardWraps(),
|
|
html.WithXHTML(),
|
|
),
|
|
)
|
|
|
|
var buf bytes.Buffer
|
|
if err := md.Convert([]byte(markdown), &buf); err != nil {
|
|
return "", fmt.Errorf("error converting markdown to HTML: %w", err)
|
|
}
|
|
|
|
return buf.String(), nil
|
|
}
|
|
|
|
// CleanJSONResponse removes markdown code blocks from LLM responses.
|
|
// This handles cases where LLMs wrap JSON in ```json ... ``` blocks despite explicit instructions.
|
|
func CleanJSONResponse(response string) string {
|
|
response = strings.TrimSpace(response)
|
|
|
|
// Handle ```json ... ``` blocks
|
|
if strings.HasPrefix(response, "```json") && strings.HasSuffix(response, "```") {
|
|
cleaned := strings.TrimPrefix(response, "```json")
|
|
cleaned = strings.TrimSuffix(cleaned, "```")
|
|
return strings.TrimSpace(cleaned)
|
|
}
|
|
|
|
// Handle generic ``` ... ``` blocks
|
|
if strings.HasPrefix(response, "```") && strings.HasSuffix(response, "```") {
|
|
cleaned := strings.TrimPrefix(response, "```")
|
|
cleaned = strings.TrimSuffix(cleaned, "```")
|
|
return strings.TrimSpace(cleaned)
|
|
}
|
|
|
|
return response
|
|
}
|