Allow disabling fancy comment text formatting

backend/app/cmd/server.go

@@ -59,34 +59,35 @@ type ServerCommand struct {
 	SSL        SSLGroup        `group:"ssl" namespace:"ssl" env-namespace:"SSL"`
 	ImageProxy ImageProxyGroup `group:"image-proxy" namespace:"image-proxy" env-namespace:"IMAGE_PROXY"`
 
-	Sites            []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
-	AnonymousVote    bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
-	AdminPasswd      string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
-	BackupLocation   string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
-	MaxBackupFiles   int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
-	LegacyImageProxy bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
-	MaxCommentSize   int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
-	MaxVotes         int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
-	RestrictVoteIP   bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
-	DurationVoteIP   time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
-	LowScore         int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
-	CriticalScore    int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
-	PositiveScore    bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
-	ReadOnlyAge      int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
-	EditDuration     time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
-	AdminEdit        bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
-	Port             int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
-	Address          string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
-	WebRoot          string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
-	UpdateLimit      float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
-	RestrictedWords  []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
-	RestrictedNames  []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
-	EnableEmoji      bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
-	SimpleView       bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
-	ProxyCORS        bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
-	AllowedHosts     []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
-	SubscribersOnly  bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
-	DisableSignature bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	Sites                      []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
+	AnonymousVote              bool          `long:"anon-vote" env:"ANON_VOTE" description:"enable anonymous votes (works only with VOTES_IP enabled)"`
+	AdminPasswd                string        `long:"admin-passwd" env:"ADMIN_PASSWD" default:"" description:"admin basic auth password"`
+	BackupLocation             string        `long:"backup" env:"BACKUP_PATH" default:"./var/backup" description:"backups location"`
+	MaxBackupFiles             int           `long:"max-back" env:"MAX_BACKUP_FILES" default:"10" description:"max backups to keep"`
+	LegacyImageProxy           bool          `long:"img-proxy" env:"IMG_PROXY" description:"[deprecated, use image-proxy.http2https] enable image proxy"`
+	MaxCommentSize             int           `long:"max-comment" env:"MAX_COMMENT_SIZE" default:"2048" description:"max comment size"`
+	MaxVotes                   int           `long:"max-votes" env:"MAX_VOTES" default:"-1" description:"maximum number of votes per comment"`
+	RestrictVoteIP             bool          `long:"votes-ip" env:"VOTES_IP" description:"restrict votes from the same ip"`
+	DurationVoteIP             time.Duration `long:"votes-ip-time" env:"VOTES_IP_TIME" default:"5m" description:"same ip vote duration"`
+	LowScore                   int           `long:"low-score" env:"LOW_SCORE" default:"-5" description:"low score threshold"`
+	CriticalScore              int           `long:"critical-score" env:"CRITICAL_SCORE" default:"-10" description:"critical score threshold"`
+	PositiveScore              bool          `long:"positive-score" env:"POSITIVE_SCORE" description:"enable positive score only"`
+	ReadOnlyAge                int           `long:"read-age" env:"READONLY_AGE" default:"0" description:"read-only age of comments, days"`
+	EditDuration               time.Duration `long:"edit-time" env:"EDIT_TIME" default:"5m" description:"edit window"`
+	AdminEdit                  bool          `long:"admin-edit" env:"ADMIN_EDIT" description:"unlimited edit for admins"`
+	Port                       int           `long:"port" env:"REMARK_PORT" default:"8080" description:"port"`
+	Address                    string        `long:"address" env:"REMARK_ADDRESS" default:"" description:"listening address"`
+	WebRoot                    string        `long:"web-root" env:"REMARK_WEB_ROOT" default:"./web" description:"web root directory"`
+	UpdateLimit                float64       `long:"update-limit" env:"UPDATE_LIMIT" default:"0.5" description:"updates/sec limit"`
+	RestrictedWords            []string      `long:"restricted-words" env:"RESTRICTED_WORDS" description:"words prohibited to use in comments" env-delim:","`
+	RestrictedNames            []string      `long:"restricted-names" env:"RESTRICTED_NAMES" description:"names prohibited to use by user" env-delim:","`
+	EnableEmoji                bool          `long:"emoji" env:"EMOJI" description:"enable emoji"`
+	SimpleView                 bool          `long:"simple-view" env:"SIMPLE_VIEW" description:"minimal comment editor mode"`
+	ProxyCORS                  bool          `long:"proxy-cors" env:"PROXY_CORS" description:"disable internal CORS and delegate it to proxy"`
+	AllowedHosts               []string      `long:"allowed-hosts" env:"ALLOWED_HOSTS" description:"limit hosts/sources allowed to embed comments" env-delim:","`
+	SubscribersOnly            bool          `long:"subscribers-only" env:"SUBSCRIBERS_ONLY" description:"enable commenting only for Patreon subscribers"`
+	DisableSignature           bool          `long:"disable-signature" env:"DISABLE_SIGNATURE" description:"disable server signature in headers"`
+	DisableFancyTextFormatting bool          `long:"disable-fancy-text-formatting" env:"DISABLE_FANCY_TEXT_FORMATTING" description:"disable fancy comments text formatting (replacement of quotes, dashes, fractions, etc)"`
 
 	Auth struct {
 		TTL struct {
@@ -541,7 +542,7 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 		Cache:             loadingCache,
 		NativeImporter:    &migrator.Native{DataStore: dataService},
 		DisqusImporter:    &migrator.Disqus{DataStore: dataService},
-		WordPressImporter: &migrator.WordPress{DataStore: dataService},
+		WordPressImporter: &migrator.WordPress{DataStore: dataService, DisableFancyTextFormatting: s.DisableFancyTextFormatting},
 		CommentoImporter:  &migrator.Commento{DataStore: dataService},
 		NativeExporter:    &migrator.Native{DataStore: dataService},
 		URLMapperMaker:    migrator.NewURLMapper,
@@ -576,33 +577,34 @@ func (s *ServerCommand) newServerApp(ctx context.Context) (*serverApp, error) {
 	}
 
 	srv := &api.Rest{
-		Version:               s.Revision,
-		DataService:           dataService,
-		WebRoot:               s.WebRoot,
-		WebFS:                 webFS,
-		RemarkURL:             s.RemarkURL,
-		ImageProxy:            imgProxy,
-		CommentFormatter:      commentFormatter,
-		Migrator:              migr,
-		ReadOnlyAge:           s.ReadOnlyAge,
-		SharedSecret:          s.SharedSecret,
-		Authenticator:         authenticator,
-		Cache:                 loadingCache,
-		NotifyService:         notifyService,
-		TelegramService:       telegramService,
-		SSLConfig:             sslConfig,
-		UpdateLimiter:         s.UpdateLimit,
-		ImageService:          imageService,
-		EmailNotifications:    contains("email", s.Notify.Users),
-		TelegramNotifications: contains("telegram", s.Notify.Users) && telegramService != nil,
-		EmojiEnabled:          s.EnableEmoji,
-		AnonVote:              s.AnonymousVote && s.RestrictVoteIP,
-		SimpleView:            s.SimpleView,
-		ProxyCORS:             s.ProxyCORS,
-		AllowedAncestors:      s.AllowedHosts,
-		SendJWTHeader:         s.Auth.SendJWTHeader,
-		SubscribersOnly:       s.SubscribersOnly,
-		DisableSignature:      s.DisableSignature,
+		Version:                    s.Revision,
+		DataService:                dataService,
+		WebRoot:                    s.WebRoot,
+		WebFS:                      webFS,
+		RemarkURL:                  s.RemarkURL,
+		ImageProxy:                 imgProxy,
+		CommentFormatter:           commentFormatter,
+		Migrator:                   migr,
+		ReadOnlyAge:                s.ReadOnlyAge,
+		SharedSecret:               s.SharedSecret,
+		Authenticator:              authenticator,
+		Cache:                      loadingCache,
+		NotifyService:              notifyService,
+		TelegramService:            telegramService,
+		SSLConfig:                  sslConfig,
+		UpdateLimiter:              s.UpdateLimit,
+		ImageService:               imageService,
+		EmailNotifications:         contains("email", s.Notify.Users),
+		TelegramNotifications:      contains("telegram", s.Notify.Users) && telegramService != nil,
+		EmojiEnabled:               s.EnableEmoji,
+		AnonVote:                   s.AnonymousVote && s.RestrictVoteIP,
+		SimpleView:                 s.SimpleView,
+		ProxyCORS:                  s.ProxyCORS,
+		AllowedAncestors:           s.AllowedHosts,
+		SendJWTHeader:              s.Auth.SendJWTHeader,
+		SubscribersOnly:            s.SubscribersOnly,
+		DisableSignature:           s.DisableSignature,
+		DisableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	srv.ScoreThresholds.Low, srv.ScoreThresholds.Critical = s.LowScore, s.CriticalScore

backend/app/migrator/wordpress.go

@@ -16,7 +16,8 @@ const wpTimeLayout = "2006-01-02 15:04:05"
 
 // WordPress implements Importer from WP xml
 type WordPress struct {
-	DataStore Store
+	DataStore                  Store
+	DisableFancyTextFormatting bool
 }
 
 type wpItem struct {
@@ -138,7 +139,7 @@ func (w *WordPress) convert(r io.Reader, siteID string) chan store.Comment {
 								ParentID:  comment.PID,
 								Imported:  true,
 							}
-							commentsCh <- commentFormatter.Format(c)
+							commentsCh <- commentFormatter.Format(c, w.DisableFancyTextFormatting)
 							stats.inpComments++
 							if stats.inpComments%1000 == 0 {
 								log.Printf("[DEBUG] processed %d comments", stats.inpComments)

backend/app/migrator/wordpress_test.go

@@ -24,7 +24,7 @@ func TestWordPress_Import(t *testing.T) {
 
 	dataStore := service.DataStore{Engine: b, AdminStore: admin.NewStaticStore("12345", nil, []string{}, "")}
 	defer dataStore.Close()
-	wp := WordPress{DataStore: &dataStore}
+	wp := WordPress{DataStore: &dataStore, DisableFancyTextFormatting: false}
 	size, err := wp.Import(strings.NewReader(xmlTestWP), siteID)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, size)
@@ -41,7 +41,7 @@ func TestWordPress_Import(t *testing.T) {
 	assert.Equal(t, "e8b1e92bbcf5b9bb88472f9bdb82d1b8c7ed39d6", c.User.IP)
 	ts, _ := time.Parse(wpTimeLayout, "2010-08-18 15:19:14")
 	assert.Equal(t, ts, c.Timestamp)
-	assert.Equal(t, c.Text, "<p>Mekkatorque was over in that tent up to the right</p>\n")
+	assert.Equal(t, "<p>«Mekkatorque» was over in that tent up to the right</p>\n", c.Text)
 	assert.True(t, c.Imported)
 
 	posts, err := dataStore.List(siteID, 0, 0)
@@ -54,6 +54,18 @@ func TestWordPress_Import(t *testing.T) {
 	count, err := dataStore.Count(store.Locator{URL: "https://realmenweardress.es/2010/07/do-you-rp/", SiteID: siteID})
 	assert.NoError(t, err)
 	assert.Equal(t, 3, count)
+
+	// test with DisableFancyTextFormatting
+	wp = WordPress{DataStore: &dataStore, DisableFancyTextFormatting: true}
+	size, err = wp.Import(strings.NewReader(xmlTestWP), siteID)
+	assert.NoError(t, err)
+	assert.Equal(t, 3, size)
+
+	last, err = dataStore.Last(siteID, 10, time.Time{}, adminUser)
+	assert.NoError(t, err)
+	require.Equal(t, 3, len(last), "3 comments imported")
+
+	assert.Equal(t, "<p>&#34;Mekkatorque&#34; was over in that tent up to the right</p>\n", last[0].Text)
 }
 
 func TestWordPress_Convert(t *testing.T) {
@@ -247,7 +259,7 @@ var xmlTestWP = `
 			<wp:comment_author_IP><![CDATA[128.243.253.117]]></wp:comment_author_IP>
 			<wp:comment_date><![CDATA[2010-08-18 15:19:14]]></wp:comment_date>
 			<wp:comment_date_gmt><![CDATA[2010-08-18 15:19:14]]></wp:comment_date_gmt>
-			<wp:comment_content><![CDATA[Mekkatorque was over in that tent up to the right]]></wp:comment_content>
+			<wp:comment_content><![CDATA["Mekkatorque" was over in that tent up to the right]]></wp:comment_content>
 			<wp:comment_approved><![CDATA[1]]></wp:comment_approved>
 			<wp:comment_type><![CDATA[]]></wp:comment_type>
 			<wp:comment_parent>13</wp:comment_parent>

backend/app/rest/api/rest.go

@@ -59,16 +59,17 @@ type Rest struct {
 		Low      int
 		Critical int
 	}
-	UpdateLimiter         float64
-	EmailNotifications    bool
-	TelegramNotifications bool
-	EmojiEnabled          bool
-	SimpleView            bool
-	ProxyCORS             bool
-	SendJWTHeader         bool
-	AllowedAncestors      []string // sets Content-Security-Policy "frame-ancestors ..."
-	SubscribersOnly       bool
-	DisableSignature      bool // prevent signature from being added to headers
+	UpdateLimiter              float64
+	EmailNotifications         bool
+	TelegramNotifications      bool
+	EmojiEnabled               bool
+	SimpleView                 bool
+	ProxyCORS                  bool
+	SendJWTHeader              bool
+	AllowedAncestors           []string // sets Content-Security-Policy "frame-ancestors ..."
+	SubscribersOnly            bool
+	DisableSignature           bool // prevent signature from being added to headers
+	DisableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 
 	SSLConfig   SSLConfig
 	httpsServer *http.Server
@@ -369,16 +370,17 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) {
 	}
 
 	privGrp := private{
-		dataService:      s.DataService,
-		cache:            s.Cache,
-		imageService:     s.ImageService,
-		commentFormatter: s.CommentFormatter,
-		readOnlyAge:      s.ReadOnlyAge,
-		authenticator:    s.Authenticator,
-		notifyService:    s.NotifyService,
-		telegramService:  s.TelegramService,
-		remarkURL:        s.RemarkURL,
-		anonVote:         s.AnonVote,
+		dataService:                s.DataService,
+		cache:                      s.Cache,
+		imageService:               s.ImageService,
+		commentFormatter:           s.CommentFormatter,
+		readOnlyAge:                s.ReadOnlyAge,
+		authenticator:              s.Authenticator,
+		notifyService:              s.NotifyService,
+		telegramService:            s.TelegramService,
+		remarkURL:                  s.RemarkURL,
+		anonVote:                   s.AnonVote,
+		disableFancyTextFormatting: s.DisableFancyTextFormatting,
 	}
 
 	admGrp := admin{

backend/app/rest/api/rest_private.go

@@ -34,16 +34,17 @@ import (
 )
 
 type private struct {
-	dataService      privStore
-	cache            LoadingCache
-	readOnlyAge      int
-	commentFormatter *store.CommentFormatter
-	imageService     *image.Service
-	notifyService    *notify.Service
-	authenticator    *auth.Service
-	telegramService  telegramService
-	remarkURL        string
-	anonVote         bool
+	dataService                privStore
+	cache                      LoadingCache
+	readOnlyAge                int
+	commentFormatter           *store.CommentFormatter
+	imageService               *image.Service
+	notifyService              *notify.Service
+	authenticator              *auth.Service
+	telegramService            telegramService
+	remarkURL                  string
+	anonVote                   bool
+	disableFancyTextFormatting bool // disables SmartyPants in the comment text rendering of the posted comments
 }
 
 // telegramService is a subset of Telegram service used for setting up user telegram notifications
@@ -88,7 +89,7 @@ func (s *private) previewCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 	comment.Sanitize()
 
 	// check if images are valid, omit proxied images as they are lazy-loaded
@@ -128,7 +129,7 @@ func (s *private) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "invalid comment", rest.ErrCommentValidation)
 		return
 	}
-	comment = s.commentFormatter.Format(comment)
+	comment = s.commentFormatter.Format(comment, s.disableFancyTextFormatting)
 
 	// check if images are valid, omit proxied images as they are lazy-loaded
 	for _, id := range s.imageService.ExtractNonProxiedPictures(comment.Text) {
@@ -212,7 +213,7 @@ func (s *private) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 	}
 
 	editReq := service.EditRequest{
-		Text:    s.commentFormatter.FormatText(edit.Text),
+		Text:    s.commentFormatter.FormatText(edit.Text, s.disableFancyTextFormatting),
 		Orig:    edit.Text,
 		Summary: edit.Summary,
 		Delete:  edit.Delete,