SAS: Major improvements (#1981)

This makes various improvements to the SAS language definition such as adding support for embedded SQL, missing keyword and language elements, more granular tokenizing, and many more.

components.json

@@ -816,6 +816,7 @@
 		},
 		"sas": {
 			"title": "SAS",
+			"require": "sql",
 			"owner": "Golmote"
 		},
 		"sass": {

components/prism-sas.js

@@ -1,34 +1,184 @@
-Prism.languages.sas = {
-	'datalines': {
-		pattern: /^\s*(?:(?:data)?lines|cards);[\s\S]+?(?:\r?\n|\r);/im,
-		alias: 'string',
-		inside: {
-			'keyword': {
-				pattern: /^(\s*)(?:(?:data)?lines|cards)/i,
-				lookbehind: true
-			},
-			'punctuation': /;/
-		}
-	},
-	'comment': [
+(function (Prism) {
+
+	var stringPattern = /(["'])(?:\1\1|(?!\1)[\s\S])*\1(?!\1)/.source;
+
+	var number = /\b(?:\d[\da-f]*x|\d+(?:\.\d+)?(?:e[+-]?\d+)?)\b/i;
+	var numericConstant = {
+		pattern: RegExp(stringPattern + '[bx]'),
+		alias: 'number'
+	};
+
+	var step = {
+		pattern: /(^|\s+)(?:proc\s+\w+|quit|run|data(?!\=))\b/i,
+		alias: 'keyword',
+		lookbehind: true
+	};
+
+	var comment = [
 		{
-			pattern: /(^\s*|;\s*)\*.*;/m,
+			pattern: /(^\s*|;\s*)\*[^;]*;/m,
 			lookbehind: true
 		},
 		/\/\*[\s\S]+?\*\//
-	],
-	'datetime': {
-		// '1jan2013'd, '9:25:19pm't, '18jan2003:9:27:05am'dt
-		pattern: /'[^']+'(?:dt?|t)\b/i,
-		alias: 'number'
-	},
-	'string': {
-		pattern: /(["'])(?:\1\1|(?!\1)[\s\S])*\1/,
+	];
+
+	var string = {
+		pattern: RegExp(stringPattern),
 		greedy: true
-	},
-	'keyword': /\b(?:data|else|format|if|input|proc\s\w+|quit|run|then|libname|set|output|options)\b/i,
-	// Decimal (1.2e23), hexadecimal (0c1x)
-	'number': /\b(?:[\da-f]+x|\d+(?:\.\d+)?(?:e[+-]?\d+)?)/i,
-	'operator': /\*\*?|\|\|?|!!?|¦¦?|<[>=]?|>[<=]?|[-+\/=&]|[~¬^]=?|\b(?:eq|ne|gt|lt|ge|le|in|not)\b/i,
-	'punctuation': /[$%@.(){}\[\];,\\]/
-};
+	};
+
+	var punctuation = /[$%@.(){}\[\];,\\]/;
+
+	var args = {
+		'arg-value': {
+			pattern: /(=)[A-Z]+/i,
+			lookbehind: true
+		},
+		'operator': /=/,
+		'arg': {
+			pattern: /[A-Z]+/i,
+			alias: 'keyword'
+		},
+		'number': number,
+		'numeric-constant': numericConstant,
+		'punctuation': punctuation,
+		'string': string
+	};
+
+	var globalStatements = {
+		pattern: /((?:^|[\s])=?)(?:catname|checkpoint execute_always|dm|endsas|filename|footnote|%include|libname|%list|lock|missing|options|page|resetline|%run|sasfile|skip|sysecho|title\d?)\b/i,
+		lookbehind: true,
+		alias: 'keyword'
+	};
+
+	Prism.languages.sas = {
+		'datalines': {
+			pattern: /^(\s*)(?:(?:data)?lines|cards);[\s\S]+?^;/im,
+			lookbehind: true,
+			alias: 'string',
+			inside: {
+				'keyword': {
+					pattern: /^(?:(?:data)?lines|cards)/i
+				},
+				'punctuation': /;/
+			}
+		},
+
+		'proc-sql': {
+			pattern: /(^proc\s+(?:fed)?sql(?:\s+[\w|=]+)?;)[\s\S]+?(?=^(?:proc\s+\w+|quit|run|data);|(?![\s\S]))/im,
+			lookbehind: true,
+			inside: {
+				'sql': {
+					pattern: RegExp(/^[ \t]*(?:select|alter\s+table|(?:create|describe|drop)\s+(?:index|table(?:\s+constraints)?|view)|create\s+unique\s+index|insert\s+into|update)(?:<str>|[^;"'])+;/.source.replace(/<str>/g, stringPattern), 'im'),
+					alias: 'language-sql',
+					inside: Prism.languages.sql
+				},
+				'global-statements': globalStatements,
+				'sql-statements': {
+					pattern: /(^|\s)(?:disconnect\s+from|exec(?:ute)?|begin|commit|rollback|reset|validate)\b/i,
+					lookbehind: true,
+					alias: 'keyword'
+				},
+				'number': number,
+				'numeric-constant': numericConstant,
+				'punctuation': punctuation,
+				'string': string
+			}
+		},
+
+		'proc-args': {
+			pattern: RegExp(/(^proc\s+\w+\s+)(?!\s)(?:[^;"']|<str>)+;/.source.replace(/<str>/g, stringPattern), 'im'),
+			lookbehind: true,
+			inside: args
+		},
+
+		/*Special keywords within macros*/
+		'macro-keyword': {
+			pattern: /((?:^|\s)=?)%(?:ABORT|BQUOTE|BY|CMS|COPY|DISPLAY|DO|ELSE|END|EVAL|GLOBAL|GO|GOTO|IF|INC|INCLUDE|INDEX|INPUT|KTRIM|LENGTH|LET|LIST|LOCAL|NRBQUOTE|NRQUOTE|NRSTR|PUT|QKTRIM|QSCAN|QSUBSTR|QSYSFUNC|QUOTE|QUPCASE|RETURN|RUN|SCAN|STR|SUBSTR|SUPERQ|SYMDEL|SYMGLOBL|SYMLOCAL|SYMEXIST|SYSCALL|SYSEVALF|SYSEXEC|SYSFUNC|SYSGET|SYSRPUT|THEN|TO|TSO|UNQUOTE|UNTIL|UPCASE|WHILE|WINDOW)\b/i,
+			lookbehind: true,
+			alias: 'keyword'
+		},
+		'macro-declaration': {
+			pattern: /^%macro[^;]+(?=;)/im,
+			inside: {
+				'keyword': /%macro/i,
+			}
+		},
+		'macro-end': {
+			pattern: /^%mend[^;]+(?=;)/im,
+			inside: {
+				'keyword': /%mend/i,
+			}
+		},
+		/*%_zscore(headcir, _lhc, _mhc, _shc, headcz, headcpct, _Fheadcz); */
+		'macro': {
+			pattern: /%_\w+(?=\()/,
+			alias: 'keyword'
+		},
+		'input': {
+			pattern: /\binput\s+[-\w\s/*.$&]+;/i,
+			inside: {
+				'input': {
+					alias: 'keyword',
+					pattern: /^input/i,
+				},
+				'comment': comment,
+				'number': number,
+				'numeric-constant': numericConstant
+			}
+		},
+		'comment': comment,
+		'options-args': {
+			pattern: /(^options)[-'"|/\\<>*+=:()\w\s]*(?=;)/im,
+			lookbehind: true,
+			inside: args
+		},
+		'function': {
+			pattern: /%?\w+(?=\()/,
+			alias: 'keyword'
+		},
+		'format': {
+			pattern: /\b(?:format|put)\b=?[\w'$.]+/im,
+			inside: {
+				'keyword': /^(?:format|put)(?=\=)/i,
+				'equals': /=/,
+				'format': {
+					pattern: /(?:\w|\$\d)+\.\d?/i,
+					alias: 'number'
+				}
+			}
+		},
+		'altformat': {
+			pattern: /\b(?:format|put)\s+[\w']+(?:\s+[$.\w]+)+(?=;)/i,
+			inside: {
+				'keyword': /^(?:format|put)/i,
+				'format': {
+					pattern: /[\w$]+\.\d?/,
+					alias: 'number'
+				}
+			}
+		},
+		'numeric-constant': numericConstant,
+		'datetime': {
+			// '1jan2013'd, '9:25:19pm't, '18jan2003:9:27:05am'dt
+			pattern: RegExp(stringPattern + '(?:dt?|t)'),
+			alias: 'number'
+		},
+		'string': string,
+		'step': step,
+		'keyword': {
+			pattern: /((?:^|\s)=?)(?:action|after|analysis|and|array|barchart|barwidth|begingraph|by|cas|cbarline|cfill|close|column|computed?|contains|data(?=\=)|define|document|do\s+over|do|dol|drop|dul|end|entryTitle|else|endcomp|fill(?:attrs)?|filename|group(?:by)?|headline|headskip|histogram|if|infile|keep|label|layout|legendlabel|length|libname|merge|midpoints|name|noobs|nowd|ods|options|or|out(?:put)?|overlay|plot|ranexp|rannor|rbreak|retain|set|session|sessref|statgraph|sum|summarize|table|temp|then\sdo|then|title\d?|to|var|where|xaxisopts|yaxisopts|y2axisopts)\b/i,
+			lookbehind: true,
+		},
+		// In SAS Studio syntax highlighting, these operators are styled like keywords
+		'operator-keyword': {
+			pattern: /\b(?:eq|ne|gt|lt|ge|le|in|not)\b/i,
+			alias: 'operator'
+		},
+		// Decimal (1.2e23), hexadecimal (0c1x)
+		'number': number,
+		'operator': /\*\*?|\|\|?|!!?|¦¦?|<[>=]?|>[<=]?|[-+\/=&]|[~¬^]=?/i,
+		'punctuation': punctuation
+	};
+
+}(Prism));

examples/prism-sas.html

@@ -6,7 +6,7 @@ <h2>Comments</h2>
 
 <h2>Numbers, dates and times</h2>
 <pre><code>42; 4.5; 4.5e-10; -3; -3.5e2; -4.2e-23;
-0afx; 0123x; abcdefx;
+0afx; 0123x;
 '1jan2013'd; '01jan09'd;
 '9:25't; '9:25:19pm't;
 '01may12:9:30:00'dt; '18jan2003:9:27:05am'dt;
@@ -31,7 +31,8 @@ <h2>Operators</h2>
 not a;</code></pre>
 
 <h2>More examples</h2>
-<pre><code>/* Some examples adapted from the documentation (http://support.sas.com/documentation/cdl/en/basess/64003/PDF/default/basess.pdf) */
+<pre><code>/* Some examples adapted from the documentation
+   http://support.sas.com/documentation/cdl/en/basess/64003/PDF/default/basess.pdf */
 
 data city; * another inline comment;
 

plugins/autoloader/prism-autoloader.js

@@ -91,6 +91,7 @@
 		],
 		"reason": "clike",
 		"ruby": "clike",
+		"sas": "sql",
 		"sass": "css",
 		"scss": "css",
 		"scala": "java",

tests/languages/sas/datalines_feature.test

@@ -35,4 +35,4 @@ baz
 
 ----------------------------------------------------
 
-Checks for datalines.
+Checks for datalines.