Syntax highlighting for ObjectScript

Did you know that Caché (and now InterSystems IRIS) has available parser for ObjectScript ?

Well, technically, lexer. It hides under the name of %SyntaxColor class. This class provides API to the DLL used by Studio to do syntax highlighting. As a result if works only under Windows.

Basic usage of this class is to generate HTML with syntax-highlighted ObjectScript.

For example, following program (taken from class reference for %SyntaxColor):

Set instr=##class(%Stream.TmpCharacter).%New()
Do instr.WriteLine(" set a = 1")
Do instr.WriteLine(" &sql(insert into sometable(A) values(:a))")
Set outstr=##class(%Stream.TmpCharacter).%New()
Set colorer=##class(%SyntaxColor).%New()
Set ret=colorer.Color(instr,outstr,"COS","PF",,,.langs,.coloringerrors)
If 'ret {Write "Fatal error: ",colorer.DLLResultCode,! Quit}
If coloringerrors {
  Write "Syntax error(s)",!
}
Do outstr.Rewind()
While 'outstr.AtEnd {
  Write outstr.ReadLine(),!
}

Generates following output:

USER>do ^test
&nbsp;<FONT COLOR="#0000ff">set&nbsp;</FONT><FONT COLOR="#800000">a&nbsp;</FONT><FONT COLOR="#000000">=&nbsp;1<BR>
&nbsp;</FONT><FONT COLOR="#800080">&amp;sql(</FONT><FONT COLOR="#0000ff">insert&nbsp;</FONT><FONT COLOR="#000080">into&nbsp;</FONT><FONT COLOR="#008000">sometable</FONT><FONT COLOR="#000000">(</FONT><FONT COLOR="#008000">A</FONT><FONT COLOR="#000000">)&nbsp;</FONT><FONT COLOR="#000080">values</FONT><FONT COLOR="#000000">(</FONT><FONT COLOR="#800000">:a</FONT><FONT COLOR="#000000">)</FONT><FONT COLOR="#800080">)</FONT>

That results in following HTML:

screnshot

Here in that sample program instr is the stream with the source code we want to highlight, outstr – resulting stream. “COS” – the language, ObjectScript (other values are “JS”, ”HTML”). The most interesting part in the call is fourth argument – “PF”. Here “P” means “don’t add enclosing <PRE> element”, and “F” – “no enclosing element with background and foreground”. For more available options – see class reference for method Color.

There is a small test page available on InterSystems Open Exchange as cache-objectscript-syntax-colorer, that uses %SyntaxColor to create highlighted ObjectScript code.

The most interesting part of %SyntaxColor, however, is that it generates not only HTML, but also XML.

For example, if in previous program we replace “PF” with “Q=N”, then result would be following:

USER>do ^test
<color>
 <line>
  <WhiteSpace> </WhiteSpace>
  <Command>set</Command>
  <WhiteSpace> </WhiteSpace>
  <Localvariable>a</Localvariable>
  <WhiteSpace> </WhiteSpace>
  <Operator>=</Operator>
  <WhiteSpace> </WhiteSpace>
  <Number>1</Number>
 </line>
 <line>
  <WhiteSpace> </WhiteSpace>
  <SQL>&amp;</SQL>
  <SQL>sql</SQL>
  <SQL>(</SQL>
  <Statementkeyword>insert</Statementkeyword>
  <WhiteSpace> </WhiteSpace>
  <Qualifierkeyword>into</Qualifierkeyword>
  <WhiteSpace> </WhiteSpace>
  <Identifier>sometable</Identifier>
  <Delimiter>(</Delimiter>
  <Identifier>A</Identifier>
  <Delimiter>)</Delimiter>
  <WhiteSpace> </WhiteSpace>
  <Qualifierkeyword>values</Qualifierkeyword>
  <Delimiter>(</Delimiter>
  <Hostvariablename>:a</Hostvariablename>
  <Delimiter>)</Delimiter>
  <SQL>)</SQL>
 </line>

And that is much more interesting, having this output we can analyze text of our ObjectScript programs. For example, find places in the code that violate Code guidelines and use shortcut names.

Another use case is determining if this particular line of ObjectScript is executable or just comment or whitespace. That can be useful for Unit Test coverage – see “Unit Test Coverage” presentation by @Timothy Leavitt from Global Summit 2018 (%SyntaxColor is mentioned at 31:10). Unit Test coverage tool is also available on Open Exchange.

If you create something interesting with this class – let us know in comments!

Comments

I found it many years ago. And last versions contains even much more than just a colorer. The complete parser also available there, but hidden. It would be good if InterSystems will open it to use for everybody. What would be great if InterSystems will make LanguageServer based on this parser?

ROUTINE ColorJSON

#include %occStatus

    // demonstration of using %SyntaxColor to output coloring information in JSON format
    
    // example code to be syntax checked
    // - lines for the syntax checker must begin with space or tab if there is no label
    Set n=0
    Set code($I(n))=" Set sum=0"
    Set code($I(n))=" For i=1:1 {"
    Set code($I(n))="  Read ""Number (return to finish): "",in,!"
    Set code($I(n))="  If in="""" {Quit}"
    Set code($I(n))="  Set sum=sum+in"
    Set code($I(n))=" }"
    Set code($I(n))=" Write ""Sum is: "",sum,!"
    Set code($I(n))=" Quit"
    
    // list code to current device
    Write !,"Code:",!
    For lineno=1:1:n {
        Write lineno,": ",code(lineno),!
    }
    Write !,"--",!!
    
    // write code to a stream and rewind it for reading
    Set input=##class(%GlobalCharacterStream).%New() If '$IsObject(input) {$$$ThrowStatus(%objlasterror)}
    For lineno=1:1:n {
        $$$THROWONERROR(sc,input.WriteLine(code(lineno)))
    }
    $$$THROWONERROR(sc,input.Rewind())
        
    // stream for coloring-output
    Set output=##class(%GlobalCharacterStream).%New() If '$IsObject(output) {$$$ThrowStatus(%objlasterror)}
    
    // create syntax checker
    Set syn=##class(%SyntaxColor).%New() If '$IsObject(syn) {$$$ThrowStatus(%objlasterror)}
    
    // top-level language (moniker) of source code
    Set language="COS"
    
    // flags for JSON output ("K" can't be used with "C" or "H")
    Set flags="K"
    
    // invoke syntax checker
    // - the Color method returns 1 for OK, 0 for ERROR (same applies to the Languages and Attributes methods) 
    If 'syn.Color(input,output,language,flags,,,,.coloringerrors) {
        Throw ##class(%Exception.General).%New("- error calling %SyntaxColor:Color: "_syn.DLLResultCode) // error details are in the DLLResultCode property
    }

    If coloringerrors {
        Write "(coloring errors)",!!
    }
    
    Do ShowJSON(syn,output)
    
    Quit

    
ShowJSON(syn,stream)
{
    Write "Raw JSON:",!
    $$$THROWONERROR(sc,stream.Rewind())
    While 'stream.AtEnd {
        Write stream.ReadLine(,.sc),! If $$$ISERR(sc) {$$$ThrowStatus(sc)}
    }
    Write !,"--",!!

    Write "Decoded JSON:",!
    Write " - format is: OFFSET(COUNT) LANGUAGE:ATTRIBUTE",!
    $$$THROWONERROR(sc,stream.Rewind())
    Set json={}.%FromJSON(stream.Read(,.sc)) If $$$ISERR(sc) {$$$ThrowStatus(sc)}
    
    // for each line ..
    // (the %GetNext loops could alternatively be done using a For loop starting at 0) 
    Set linesiter=json.%GetIterator()
    While linesiter.%GetNext(.lineno,.linejson) {
    
        Write !,"Line ",lineno+1,! // this array is numbered from 0 so we need to add 1 to get a line number corresponding to the code listing
        
        // for each colored item in the line
        Set onelineiter=linejson.%GetIterator()
        While onelineiter.%GetNext(,.item) {
            Write " ",$$ShowItem(syn,item),!
        }        
    }
    Write !,"--",!!
}


ShowItem(syn,item)
{
    Quit item.p_"("_item.c_") "_$$Language(syn,item.l)_":"_$$Attribute(syn,item.l,item.s)
}


Language(syn,langindex)
{
    // in a real program this should be cached
    If 'syn.Languages(.languages) {Throw ##class(%Exception.General).%New("- error calling %SyntaxColor:Languages: "_syn.DLLResultCode)}
    Quit $List(languages,langindex+1) // first language index is 0 so we need to add 1 to langindex
}


Attribute(syn,langindex,attrindex)
{
    // in a real program this should be cached
    If 'syn.Attributes(langindex,.attributes) {Throw ##class(%Exception.General).%New("- error calling %SyntaxColor:Attributes: "_syn.DLLResultCode)}
    Quit $List(attributes,attrindex+1) // first attribute index is 0 so we need to add 1 to attrindex
}