Here's something that scala folks should probably use more often Option.isDefined Sometimes its much simpler to get something that returns an Option, and then simply test isDefined. Rather that doing a full blown match. For example if( foo().isDefined() ){ doSomething() } else { doSomethingElse() } rather than foo() match { case Some( o : Object ) => { doSomething() } case _ => { doSomethingElse() } } I think you can argue over readability. But I think the first example emphasizes the intent that the programmer is reacting to the point that foo is returning something vs. returning nothing (or None). Also, one point in the first one. Sometimes the programmer doesn't really care what was returned by foo.

Grep some more

Here is an update to the Java grep. I am doing this as I follow the Mastering Regular Expressions book to that I can test the java examples. And hopefully make the Java examples less awkward.

its a little cleaner than the last one.

package org.rip.regex;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;

public class grep {

public grep() {

grepPatterns = new ArrayList< Pattern >();
filesToProcess = new ArrayList< File >();
excludes = new ArrayList< String >();
regexes = new ArrayList< String >();
}

private List< Pattern > grepPatterns;

private List< String > regexes;

private List< File > filesToProcess;

private ArrayList< String > excludes;

public Pattern compilePattern( final String pat ) {

int flags = 0;

String pattern = new String( pat );

if( ignoreCase ) {
flags |= Pattern.CASE_INSENSITIVE;
}

// we are ignoring line-regex if both are supplied
if( wordRegex ) {
// poor mans way to do it
pattern = "\\b" + pattern + "\\b";
}
else {
if( lineRegex ) {

// poor mans way to do it
pattern = "^" + pattern + "$";
}
}

return Pattern.compile( pattern, flags );

}


private void printMessage( final String msg ) {

if( !quiet ) {
System.out.println( msg );
}
}

private void grepFiles() {

// at this point the list was expanded into file names
for( final File f : filesToProcess ) {

grepFile( f );
}

}

private void transferOptions( final CommandLine commandLine ) {

long context = 0;

if( commandLine.hasOption( 'C' ) || commandLine.hasOption( "context" ) ) {
String s = commandLine.getOptionValue( 'C' );
context = Long.parseLong( s );
}


if( commandLine.hasOption( 'A' ) || commandLine.hasOption( "after-context" ) ) {
String s = commandLine.getOptionValue( 'A' );
afterContext = Long.parseLong( s );
}
else {
afterContext = context;
}

if( commandLine.hasOption( 'B' ) || commandLine.hasOption( "before-context" ) ) {
String s = commandLine.getOptionValue( 'B' );
beforeContext = Long.parseLong( s );
}
else {
beforeContext = context;
}

if( commandLine.hasOption( 'i' ) || commandLine.hasOption( "ignore-case" ) ) {
ignoreCase = true;
}

if( commandLine.hasOption( 'v' ) || commandLine.hasOption( "invert-match" ) ) {
invertMatch = true;
}

maxCount = 0;
if( commandLine.hasOption( 'm' ) || commandLine.hasOption( "max-count" ) ) {
String s = commandLine.getOptionValue( 'm' );
maxCount = Long.parseLong( s );
}

if( commandLine.hasOption( 'l' ) || commandLine.hasOption( "files-with-matches" ) ) {
printFileNameOnly = true;
}

if( commandLine.hasOption( 'b' ) || commandLine.hasOption( "byte-offset" ) ) {
printByteOffset = true;
}

if( commandLine.hasOption( 'q' ) || commandLine.hasOption( "--quiet" ) || commandLine.hasOption( "silent" ) ) {
quiet = true;
}

if( commandLine.hasOption( 'c' ) || commandLine.hasOption( "count" ) ) {
printCountOnly = true;
}

printFilesWithoutMatch = commandLine.hasOption( 'L' ) || commandLine.hasOption( "files-without-match" );

wordRegex = commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" );

lineRegex = commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" );

if( commandLine.hasOption( "exclude-from" ) ) {
readExcludeFrom( commandLine.getOptionValue( "exclude-from" ) );
}

// this could be specified multiple times
if( commandLine.hasOption( 'e' ) || commandLine.hasOption( "regexp" ) ) {
regexes.add( commandLine.getOptionValue( 'e' ) );
}

if( commandLine.hasOption( 'f' ) || commandLine.hasOption( "file" ) ) {
readRegexFromFile( commandLine.getOptionValue( 'f' ) );
}

noMessages = commandLine.hasOption( 's' ) || commandLine.hasOption( "no-messages" );

if( commandLine.hasOption( 'H' ) || commandLine.hasOption( "with-filename" ) ) {
printFileName = true;
}

if( commandLine.hasOption( 'h' ) || commandLine.hasOption( "no-filename" ) ) {
printFileName = false;
}

printMatchOnly = commandLine.hasOption( 'o' ) || commandLine.hasOption( "only-matching" );

printLineNumber = commandLine.hasOption( 'n' ) || commandLine.hasOption( "line-number" );

recurseDirectories = false;

if( commandLine.hasOption( 'd' ) || commandLine.hasOption( "directories" ) ) {

String val = commandLine.getOptionValue( 'd' );
if( StringUtils.endsWithIgnoreCase( val, "recurse" ) ) {
recurseDirectories = true;
}

}

if( commandLine.hasOption( 'R' )
|| commandLine.hasOption( 'r' )
|| commandLine.hasOption( "recursive" ) ) {
recurseDirectories = true;
}

if( commandLine.hasOption( "include" ) ) {
includeFilePattern = commandLine.getOptionValue( "include" );
}

if( commandLine.hasOption( "exclude" ) ) {
excludeFilePattern = commandLine.getOptionValue( "exclude" );
}

if( commandLine.hasOption( "exclude-dir" ) ) {
excludeDirPattern = commandLine.getOptionValue( "exclude-dir" );
}

}

private boolean invertMatch = false;
private boolean ignoreCase = false;
private long maxCount;

private boolean printFileNameOnly = false;

private boolean printByteOffset = false;

private boolean quiet = false;

private boolean printCountOnly = false;

private boolean printFilesWithoutMatch = false;

private boolean wordRegex = false;

private boolean lineRegex = false;

private boolean noMessages = false;

private boolean printFileName = true;

private boolean printMatchOnly = false;
private boolean printLineNumber=false;

private boolean recurseDirectories = false;

private String includeFilePattern = null;

private String excludeFilePattern = null;

private String excludeDirPattern = null;

public static void main( final String[] args ) {

grep theGrep = new grep();
grepCommandLine cmd = new grepCommandLine();

if( ArrayUtils.isEmpty( args ) ) {
grepCommandLine.usage();
return;
}

// parse the command line
try {
final CommandLineParser parser = new PosixParser();
cmd.setCommandLine( parser.parse( grepCommandLine.grepOptions, args ) );
}
catch( final ParseException pe ) {
System.out.print( pe.getMessage() );
grepCommandLine.usage();
return;
}

theGrep.transferOptions( cmd.getCommandLine() );

// if version or help was given then print and quit
if( cmd.getCommandLine().hasOption( "help" ) ) {
grepCommandLine.usage();
return;
}

if( cmd.getCommandLine().hasOption( "version" ) || cmd.getCommandLine().hasOption( 'V' ) ) {
grepCommandLine.version();
return;
}

//
if( cmd.getCommandLine().hasOption( 'e' ) || cmd.getCommandLine().hasOption( "regexp" ) ) {
if( ArrayUtils.isEmpty( cmd.getCommandLine().getOptionValues( 'e' ) ) ) {
grepCommandLine.usage();
return;
}
}

if( cmd.getCommandLine().hasOption( 'f' ) || cmd.getCommandLine().hasOption( "file" ) ) {
if( ArrayUtils.isEmpty( cmd.getCommandLine().getOptionValues( 'f' ) ) ) {
grepCommandLine.usage();
return;
}
}

if( cmd.getCommandLine().hasOption( "exclude-from" ) ) {
String s = cmd.getCommandLine().getOptionValue( "exclude-from" );
if( StringUtils.isEmpty( s ) ) {
grepCommandLine.usage();
return;
}
}

// get the unprocessed items
final List argList = cmd.getCommandLine().getArgList();

// we must have some args besides options
if( CollectionUtils.isEmpty( argList ) ) {
// TODO - regex could be passed from file or arg
// input could be from STDIN
grepCommandLine.usage();
return;
}

// just to get things started
// we are assuming: [OPTION]... PATTERN [FILE]...
// options are pulled out by the command line processor
// if the regex was not supplied via file or arg
// we assume its the first arg
// if the regex was set via arg or file
// we assume the first arg is a file
int fileOffset = 0;
if( CollectionUtils.isEmpty( theGrep.regexes ) ) {
theGrep.regexes.add( (String)argList.get( 0 ) );
fileOffset++;
}

// make sure its a good regex
try {
theGrep.compilePatterns();
}
catch( final PatternSyntaxException pse ) {
theGrep.printErrorMessage( "invalid regex syntax\n" + pse.getMessage() );
return;
}

if( CollectionUtils.isEmpty( theGrep.grepPatterns ) ) {
grepCommandLine.usage();
return;
}

// just to get things going
List< String > files = argList.subList( fileOffset, argList.size() );
if( files.size() > 0 ) {
theGrep.printFileName = true;
}
theGrep.processFileArgs( files );

// ok lets start
theGrep.grepFiles();

}

private void compilePatterns() {

for( String s : regexes ) {
grepPatterns.add( compilePattern( s ) );
}
}

private void readExcludeFrom( final String val ) {


try( BufferedReader br = new BufferedReader( new FileReader( new File( val ) ) ) ) {
excludes = new ArrayList< String >();

for( String line = br.readLine(); line != null; line = br.readLine() ) {
excludes.add( line );
}

}
catch( IOException e ) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}





public void printErrorMessage( final String msg ) {

if( !noMessages ) {
System.out.println(msg);
}
}







private void printMatch( final File file, final String line, final long lineNumber,
final long count, final long byteOffset, final List< String > beforeContextLines,
final List< String > afterContextLines ) {

if( quiet || printFilesWithoutMatch || printCountOnly ) {
return;
}

/*
* if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
* System.out.println( "--" );
* }
*/

if( beforeContext > 0 ) {
for( String bc : beforeContextLines ) {
String s = new String( bc );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

String msg = "";
if( printFileNamesOnly() ) {
try {
System.out.println( String.format( "%1$s", file.getCanonicalPath() ) );
}
catch( IOException ioe ) {
// TODO:
}
return;
}

if( printFileName ) {
try {
msg += file.getCanonicalPath() + ":";
}
catch( IOException ioe ) {}
}

if( printLineNumber ) {
msg += lineNumber + ":";
}

if( printByteOffset ) {
msg += byteOffset + ":";
}

msg = msg.length() > 1 ? msg + " " + line : line;

System.out.println( msg );

if( afterContext > 0 ) {
for( String ac : afterContextLines ) {
String s = new String( ac );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

if( ( beforeContext > 0 ) || ( afterContext > 0 ) ) {
System.out.println( "--" );
}
}



// TODO: note that file names can contain wild cards too
// so this method should also expand directories as well
private void processFileArgs( final List list ) {

filesToProcess = new ArrayList< File >();

for( final Object o : list ) {
final String name = (String)o;

final File f = new File( name );

if( f.exists() ) {
if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
filesToProcess.add( f );
}
}
}
else if( f.isDirectory() ) {
if( recurseDirectories ) {
filesToProcess.addAll( recurseDir( f ) );
}
}
}
}

}

private List< File > recurseDir( final File dir ) {

List< File > files = new ArrayList< File >();

for( File f : dir.listFiles() ) {

if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
files.add( f );
}
}
}
else if( f.isDirectory() ) {
if( !excludeDir( f ) ) {
files.addAll( recurseDir( f ) );
}
}

}

return files;

}





private void readRegexFromFile( final String fname ) {


File file = new File( fname );
if( file.exists() && file.isFile() ) {

try( BufferedReader br = new BufferedReader( new FileReader( file ) ) ) {

String line = br.readLine();
if( StringUtils.isNotBlank( line ) ) {
regexes.add( line );
}

}
catch( IOException e ) {
printErrorMessage( "Cannot open file:" + fname );
}
}


}

private void reset() {

grepPatterns = new ArrayList< Pattern >();
regexes = new ArrayList< String >();
filesToProcess = null;
excludes = null;


}





public boolean includeFile( final File f ) {

if( StringUtils.isNotEmpty( includeFilePattern ) ) {

String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, includeFilePattern );

}

return true;
}

public boolean excludeFile( final File f ) {

if( StringUtils.isNotBlank( excludeFilePattern ) ) {

String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, excludeFilePattern );
}

if( CollectionUtils.isNotEmpty( excludes ) ) {
for( String pat : excludes ) {
String nm = FilenameUtils.getName( f.getName() );
boolean ex = FilenameUtils.wildcardMatch( nm, pat );
if( ex ) {
return true;
}
}
}

return false ;

}

private boolean excludeDir( final File f ) {

if( StringUtils.isNotBlank( excludeDirPattern ) ) {
if( FilenameUtils.wildcardMatch( FilenameUtils.getName( f.getName() ), excludeDirPattern ) ) {
return true;
}
}

return false;
}

private boolean printFileNamesOnly() {

return printFilesWithoutMatch || printFileNameOnly;
}




private long beforeContext = 0;

private long afterContext = 0;

private Matcher matchAny( final String line ) {

for( Pattern pat : grepPatterns ) {

Matcher m = pat.matcher( line );
if( m.find() ) {
return m;
}
}

return null;
}

private void grepFile( final File file ) {


long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContextLines = new LinkedList< String >();
List< String > afterContextLines = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ), 16000 ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = matchAny( line );

if( ( ( m != null ) && !invertMatch ) ) {
count++;
if( afterContext > 0 ) {
afterContextLines.clear();
bfr.mark( 4000 );
for( int i = 0; i < afterContext; i++ ) { String s = bfr.readLine(); if( s != null ) { afterContextLines.add( s ); } else { break; } } bfr.reset(); } String match = line; if( printMatchOnly ) { match = m.group(); } printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContextLines, afterContextLines ); if( printFileNameOnly ) { return; } // } else if( ( m == null ) && invertMatch ) { count++; printMatch( file, line, lineNumber, count, byteOffset, beforeContextLines, afterContextLines ); // TODO: this has a slightly different meaning if( printFileNameOnly ) { return; } } if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContextLines.add( line );
if( beforeContextLines.size() > beforeContext ) {
beforeContextLines.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch ) {
printMessage( file.getName() );
}
}

if( printCountOnly ) {
printMessage( ( file.getName() + ":" + count ) );
}

}

private boolean isRegexSet() {

return CollectionUtils.isNotEmpty( regexes );
}
}



package org.rip.regex;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;

public class grepCommandLine {

static Options grepOptions;

static {
grepCommandLine.grepOptions = createOptions();
}

private CommandLine commandLine;

public grepCommandLine() {

}

public CommandLine getCommandLine() {

return commandLine;
}

public void setCommandLine( final CommandLine commandLine ) {

this.commandLine = commandLine;
}

private static Options createOptions() {

final Options opts = new Options();

opts.addOption( "E", "extended-regexp", false, "PATTERN is an extended regular expression (ERE)" );
opts.addOption( "F", "fixed-strings", false, "PATTERN is a set of newline-separated fixed strings" );
opts.addOption( "G", "basic-regexp", false, "PATTERN is a basic regular expression (BRE)" );
opts.addOption( "P", "perl-regexp", false, "PATTERN is a Perl regular expression" );
opts.addOption( OptionBuilder.withLongOpt( "regexp" ).hasArg().withArgName( "PATTERN" )
.withDescription( "use PATTERN for matching" ).create( 'e' ) );
opts.addOption( OptionBuilder.withLongOpt( "file" ).hasArg().withArgName( "FILE" )
.withDescription( "obtain PATTERN from FILE" ).create( 'f' ) );
opts.addOption( "i", "ignore-case", false, "ignore case distinctions" );
opts.addOption( "w", "word-regexp", false, "force PATTERN to match only whole words" );
opts.addOption( "x", "line-regexp", false, "force PATTERN to match only whole lines" );
opts.addOption( "z", "null-data", false, "a data line ends in 0 byte, not newline" );
opts.addOption( "s", "no-messages", false, "suppress error messages" );
opts.addOption( "v", "invert-match", false, "select non-matching lines" );
opts.addOption( "V", "version", false, "print version information and exit" );
opts.addOption( OptionBuilder.withLongOpt( "help" ).withDescription( "display this help and exit" ).create() );
// opts.addOption( OptionBuilder.withLongOpt( "mmap" ).withDescription(
// "ignored for backwards compatibility" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "max-count" ).withArgName( "NUM" ).hasArg()
.withDescription( "stop after NUM matches" ).create( 'm' ) );
opts.addOption( "b", "byte-offset", false, "print the byte offset with output lines" );
opts.addOption( "n", "line-number", false, "print line number with output lines" );
opts.addOption( OptionBuilder.withLongOpt( "line-buffered" ).withDescription( "flush output on every line" )
.create() );
opts.addOption( "H", "with-filename", false, "print the filename for each match" );
opts.addOption( "h", "no-filename", false, "suppress the prefixing filename on output" );
opts.addOption( OptionBuilder.withLongOpt( "label" ).hasArg().withArgName( "LABEL" )
.withDescription( "print LABEL as filename for standard input" ).create() );
opts.addOption( "o", "only-matching", false, "show only the part of a line matching PATTERN" );
opts.addOption( "q", "quiet", false, "suppress all normal output" );
opts.addOption( OptionBuilder.withLongOpt( "silent" ).withDescription( "suppress all normal output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "binary-files" ).hasArg().withArgName( "TYPE" )
.withDescription( "assume that binary files are TYPE" ).create() );
// TYPE is `binary', `text', or `without-match'
opts.addOption( "a", "text", false, "equivalent to --binary-files=text" );
opts.addOption( "I", false, "equivalent to --binary-files=without-match" );
opts.addOption( OptionBuilder.withLongOpt( "directories" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle directories" ).create( 'd' ) );
// ACTION is `read', `recurse', or `skip'"
opts.addOption( OptionBuilder.withLongOpt( "devices" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle devices, FIFOs and sockets" ).create( 'D' ) );
// ACTION is `read' or `skip'
opts.addOption( "R", false, "equivalent to --directories=recurse" );
opts.addOption( "r", "recursive", false, "equivalent to --directories=recurse" );
opts.addOption( OptionBuilder.withLongOpt( "include" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "search only files that match FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "skip files and directories matching FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-from" ).hasArg().withArgName( "FILE" )
.withDescription( "skip files matching any file pattern from FILE" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-dir" ).hasArg().withArgName( "PATTERN" )
.withDescription( "directories that match PATTERN will be skipped" ).create() );
opts.addOption( "L", "files-without-match", false, "print only names of FILEs containing no match" );
opts.addOption( "l", "files-with-matches", false, "print only names of FILEs containing matches" );
opts.addOption( "c", "count", false, "print only a count of matching lines per FILE" );
opts.addOption( "T", "initial-tab", false, "make tabs line up (if needed)" );
opts.addOption( "Z", "null", false, "print 0 byte after FILE name" );
opts.addOption( OptionBuilder.withLongOpt( "before-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of leading context" ).create( 'B' ) );
opts.addOption( OptionBuilder.withLongOpt( "after-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of trailing context" ).create( 'A' ) );
opts.addOption( OptionBuilder.withLongOpt( "contex" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of output context" ).create( 'C' ) );
// * + " -NUM same as --context=NUM\n" +
opts.addOption( OptionBuilder.withLongOpt( "color" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "colour" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
// *
// " WHEN is `always', `never', or `auto'\n"
opts.addOption( "U", "binary", false, "do not strip CR characters at EOL (MSDOS)" );
opts.addOption( "u", "unix-byte-offsets", false, "report offsets as if CRs were not there (MSDOS)" );

// non gnu options
opts.addOption( OptionBuilder.withLongOpt( "debug" ).withDescription( "turn on debugging output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "verbose" ).withDescription( "turn on verbose output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "test" ).withDescription( "run in test mode" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "logging" ).withDescription( "use nice logging, Log4J/SLF4J" )
.create() );
opts.addOption( OptionBuilder.withLongOpt( "file-long" ).hasArg().withArgName( "FILE" )
.withDescription( "read PATTERN from file using long format" ).create() );

return opts;
}

public static void usage() {

final String msg =
"Usage: grep [OPTION]... PATTERN [FILE]...\n"
+ "Search for PATTERN in each FILE or standard input.\n"
+ "PATTERN is, by default, a basic regular expression (BRE).\n"
+ "Example: grep -i 'hello world' menu.h main.c\n"
+ "Regexp selection and interpretation:\n"
+ " -E, --extended-regexp PATTERN is an extended regular expression (ERE)\n" // no
+ " -F, --fixed-strings PATTERN is a set of newline-separated fixed strings\n" // no
+ " -G, --basic-regexp PATTERN is a basic regular expression (BRE)\n" // no
+ " -P, --perl-regexp PATTERN is a Perl regular expression\n" // no
+ " -e, --regexp=PATTERN use PATTERN for matching\n" // done
+ " -f, --file=FILE obtain PATTERN from FILE\n" // done
+ " -i, --ignore-case ignore case distinctions\n" // done
+ " -w, --word-regexp force PATTERN to match only whole words\n" // done
+ " -x, --line-regexp force PATTERN to match only whole lines\n" // done
+ " -z, --null-data a data line ends in 0 byte, not newline\n" // yes?
+ "Miscellaneous:\n"
+ " -s, --no-messages suppress error messages\n" // done
+ " -v, --invert-match select non-matching lines\n" // done
+ " -V, --version print version information and exit\n" // done
+ " --help display this help and exit\n" // done
// +
+ " --mmap ignored for backwards compatibility\n" // no
+ "Output control:\n"
+ " -m, --max-count=NUM stop after NUM matches\n" // done - max-total
+ " -b, --byte-offset print the byte offset with output lines\n" // done
+ " -n, --line-number print line number with output lines\n" // done
+ " --line-buffered flush output on every line\n" // yes
+ " -H, --with-filename print the filename for each match\n" // done
+ " -h, --no-filename suppress the prefixing filename on output\n" // done
+ " --label=LABEL print LABEL as filename for standard input\n" // yes?
+ " -o, --only-matching show only the part of a line matching PATTERN\n" // done
+ " -q, --quiet, --silent suppress all normal output\n" // done
+ " --binary-files=TYPE assume that binary files are TYPE;\n" // yes?
+ " TYPE is `binary', `text', or `without-match'\n"
+ " -a, --text equivalent to --binary-files=text\n" // yes?
+ " -I equivalent to --binary-files=without-match\n" // yes?
+ " -d, --directories=ACTION how to handle directories;\n" // done
+ " ACTION is `read', `recurse', or `skip'\n"
+ " -D, --devices=ACTION how to handle devices, FIFOs and sockets;\n" // no
+ " ACTION is `read' or `skip'\n"
+ " -R, -r, --recursive equivalent to --directories=recurse\n" // done
+ " --include=FILE_PATTERN search only files that match FILE_PATTERN\n" // done
+ " --exclude=FILE_PATTERN skip files and directories matching FILE_PATTERN\n" // done
+ " --exclude-from=FILE skip files matching any file pattern from FILE\n" // done
+ " --exclude-dir=PATTERN directories that match PATTERN will be skipped.\n" // done
+ " -L, --files-without-match print only names of FILEs containing no match\n" // done
+ " -l, --files-with-matches print only names of FILEs containing matches\n" // done
+ " -c, --count print only a count of matching lines per FILE\n" // done
+ " -T, --initial-tab make tabs line up (if needed)\n" // no
+ " -Z, --null print 0 byte after FILE name\n" // no - use --sep=CHAR
+ "Context control:\n"
+ " -B, --before-context=NUM print NUM lines of leading context\n" // done
+ " -A, --after-context=NUM print NUM lines of trailing context\n" // done
+ " -C, --context=NUM print NUM lines of output context\n" // done
+ " -NUM same as --context=NUM\n" // no
+ " --color[=WHEN],\n" // no
+ " --colour[=WHEN] use markers to highlight the matching strings;\n" // no
+ " WHEN is `always', `never', or `auto'\n"
+ " -U, --binary do not strip CR characters at EOL (MSDOS)\n" // no?
+ " -u, --unix-byte-offsets report offsets as if CRs were not there (MSDOS)\n" // no?
+ "\n"
+ " `egrep' means `grep -E'. `fgrep' means `grep -F'.\n"
+ " Direct invocation as either `egrep' or `fgrep' is deprecated.\n"
+ " With no FILE, or when FILE is -, read standard input. If less than two FILEs\n"
+ " are given, assume -h. Exit status is 0 if any line was selected, 1 otherwise;\n"
+ " if any error occurs and -q was not given, the exit status is 2.\n"
+ "\n"
+ " Report bugs to: bug-grep@gnu.org\n"
+ " GNU Grep home page: \n"
+ " General help using GNU software: \n";
System.out.print( msg );

}

public static void version() {

System.out.println( "Java Grep 0.1" );

}

}

Java Grep

This is more of the real deal command line grep.
Not some of the crap that I've seen where they just loop through a file and execute a regex search.

I'll clean up later and make things non static and possibly implement a few other command line options.

For the most part, it matches the output of GNU Grep. I even left the GNU Grep tag in the usage() function. I'll remove that later too.

And yes code formatting is terrible.


import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;

public class grep {

private static Options grepOptions;

private static CommandLine commandLine;

private static Pattern grepPattern;

private static String regex;

static {
grepOptions = createOptions();
}


private static List filesToProcess ;

private static ArrayList< String > excludes;

public static Pattern compilePattern( final String pat ) {

int flags = 0;

String pattern = new String( pat );

if( commandLine.hasOption( 'i' ) || commandLine.hasOption( "ignore-case" ) ) {
flags |= Pattern.CASE_INSENSITIVE;
}

if( ( commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" ) )
&&
( commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" ) ) ) {
printErrorMessage( "Conflicting options (-w|--word-regexp) && (-x|--line-regexp)" );
return null;
}

if( commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" ) ) {

// poor mans way to do it
pattern = "\\b" + pattern + "\\b";
}

if( commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" ) ) {

// poor mans way to do it
pattern = "^" + pattern + "$";
}

return Pattern.compile( pattern, flags );

}

private static Options createOptions() {

final Options opts = new Options();

opts.addOption( "E", "extended-regexp", false, "PATTERN is an extended regular expression (ERE)" );
opts.addOption( "F", "fixed-strings", false, "PATTERN is a set of newline-separated fixed strings" );
opts.addOption( "G", "basic-regexp", false, "PATTERN is a basic regular expression (BRE)" );
opts.addOption( "P", "perl-regexp", false, "PATTERN is a Perl regular expression" );
opts.addOption( OptionBuilder.withLongOpt( "regexp" ).hasArg().withArgName( "PATTERN" )
.withDescription( "use PATTERN for matching" ).create( 'e' ) );
opts.addOption( OptionBuilder.withLongOpt( "file" ).hasArg().withArgName( "FILE" )
.withDescription( "obtain PATTERN from FILE" ).create( 'f' ) );
opts.addOption( "i", "ignore-case", false, "ignore case distinctions" );
opts.addOption( "w", "word-regexp", false, "force PATTERN to match only whole words" );
opts.addOption( "x", "line-regexp", false, "force PATTERN to match only whole lines" );
opts.addOption( "z", "null-data", false, "a data line ends in 0 byte, not newline" );
opts.addOption( "s", "no-messages", false, "suppress error messages" );
opts.addOption( "v", "invert-match", false, "select non-matching lines" );
opts.addOption( "V", "version", false, "print version information and exit" );
opts.addOption( OptionBuilder.withLongOpt( "help" ).withDescription( "display this help and exit" ).create() );
// opts.addOption( OptionBuilder.withLongOpt( "mmap" ).withDescription(
// "ignored for backwards compatibility" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "max-count" ).withArgName( "NUM" ).hasArg()
.withDescription( "stop after NUM matches" ).create( 'm' ) );
opts.addOption( "b", "byte-offset", false, "print the byte offset with output lines" );
opts.addOption( "n", "line-number", false, "print line number with output lines" );
opts.addOption( OptionBuilder.withLongOpt( "line-buffered" ).withDescription( "flush output on every line" )
.create() );
opts.addOption( "H", "with-filename", false, "print the filename for each match" );
opts.addOption( "h", "no-filename", false, "suppress the prefixing filename on output" );
opts.addOption( OptionBuilder.withLongOpt( "label" ).hasArg().withArgName( "LABEL" )
.withDescription( "print LABEL as filename for standard input" ).create() );
opts.addOption( "o", "only-matching", false, "show only the part of a line matching PATTERN" );
opts.addOption( "q", "quiet", false, "suppress all normal output" );
opts.addOption( OptionBuilder.withLongOpt( "silent" ).withDescription( "suppress all normal output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "binary-files" ).hasArg().withArgName( "TYPE" )
.withDescription( "assume that binary files are TYPE" ).create() );
// TYPE is `binary', `text', or `without-match'
opts.addOption( "a", "text", false, "equivalent to --binary-files=text" );
opts.addOption( "I", false, "equivalent to --binary-files=without-match" );
opts.addOption( OptionBuilder.withLongOpt( "directories" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle directories" ).create( 'd' ) );
// ACTION is `read', `recurse', or `skip'"
opts.addOption( OptionBuilder.withLongOpt( "devices" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle devices, FIFOs and sockets" ).create( 'D' ) );
// ACTION is `read' or `skip'
opts.addOption( "R", false, "equivalent to --directories=recurse" );
opts.addOption( "r", "recursive", false, "equivalent to --directories=recurse" );
opts.addOption( OptionBuilder.withLongOpt( "include" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "search only files that match FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "skip files and directories matching FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-from" ).hasArg().withArgName( "FILE" )
.withDescription( "skip files matching any file pattern from FILE" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-dir" ).hasArg().withArgName( "PATTERN" )
.withDescription( "directories that match PATTERN will be skipped" ).create() );
opts.addOption( "L", "files-without-match", false, "print only names of FILEs containing no match" );
opts.addOption( "l", "files-with-matches", false, "print only names of FILEs containing matches" );
opts.addOption( "c", "count", false, "print only a count of matching lines per FILE" );
opts.addOption( "T", "initial-tab", false, "make tabs line up (if needed)" );
opts.addOption( "Z", "null", false, "print 0 byte after FILE name" );
opts.addOption( OptionBuilder.withLongOpt( "before-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of leading context" ).create( 'B' ) );
opts.addOption( OptionBuilder.withLongOpt( "after-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of trailing context" ).create( 'A' ) );
opts.addOption( OptionBuilder.withLongOpt( "contex" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of output context" ).create( 'C' ) );
// * + " -NUM same as --context=NUM\n" +
opts.addOption( OptionBuilder.withLongOpt( "color" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "colour" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
// *
// " WHEN is `always', `never', or `auto'\n"
opts.addOption( "U", "binary", false, "do not strip CR characters at EOL (MSDOS)" );
opts.addOption( "u", "unix-byte-offsets", false, "report offsets as if CRs were not there (MSDOS)" );

opts.addOption( OptionBuilder.withLongOpt( "debug" ).withDescription( "turn on debugging output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "verbose" ).withDescription( "turn on verbose output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "test" ).withDescription( "run in test mode" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "logging" ).withDescription( "use nice logging, Log4J/SLF4J" )
.create() );

return opts;
}


private static void grepFileOrig( final File file ) {

long maxCount = maxCount();
long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContext = new LinkedList< String >();
List< String > afterContext = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ) ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = grepPattern.matcher( line );
boolean found = m.find();

if( ( found && !invertMatch() ) ) {
count++;
String match = line;
if( printMatchOnly() ) {
match = m.group();
}
printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext );
if( printFileNameOnly() ) {
return;
}
//
}
else if( !found && invertMatch() ) {
count++;
printMatch( file, line, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext );
// TODO: this has a slightly different meaning
if( printFileNameOnly() ) {
return;
}
}

if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContext.add( line );
if( beforeContext.size() > beforeContext() ) {
beforeContext.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch() ) {
printMessage( file.getName() );
}
}

if( printCountOnly() ) {
printMessage( ( file.getName() + ":" + count ) );
}

}

private static void printMessage( final String msg ) {

if( !quiet() ) {
System.out.println( msg );
}
}

private static void grepFiles() {

// at this point the list was expanded into file names
for( final File f : filesToProcess ) {

grepFile( f );
}

}

public static boolean invertMatch() {
return commandLine.hasOption( 'v' ) || commandLine.hasOption( "invert-match" );
}

public static void main( final String[] args ) {

reset();

if( ArrayUtils.isEmpty( args ) ) {
usage();
return;
}

// parse the command line
try {
final CommandLineParser parser = new PosixParser();
commandLine = parser.parse( grepOptions, args );
}
catch( final ParseException pe ) {
System.out.print( pe.getMessage() );
usage();
return;
}

// if version or help was given then print and quit
if( commandLine.hasOption( "help" ) ) {
usage();
return;
}

if( commandLine.hasOption( "version" ) || commandLine.hasOption( 'V' ) ) {
version();
return;
}

//
if( commandLine.hasOption( 'e' ) || commandLine.hasOption( "regexp" ) ) {
String r = commandLine.getOptionValue( 'e' );
if( r != null ) {
regex = r;
}
else {
usage();
return;
}
}

if( commandLine.hasOption( 'f' ) || commandLine.hasOption( "file" ) ) {
if( StringUtils.isNotEmpty( regex ) ) {
// cannot specify both
usage();
return;
}

String r = readRegexFromFile();
if( StringUtils.isEmpty( r ) ) {
usage();
return;
}
regex = r;
}

if( commandLine.hasOption( "exclude-from" ) ) {
readExcludeFrom();
if( CollectionUtils.isEmpty( excludes ) ) {
usage();
return;
}
}

// get the unprocessed items
final List argList = commandLine.getArgList();

// we must have some args besides options
if( CollectionUtils.isEmpty( argList ) ) {
usage();
return;
}

// just to get things started
// we are assuming: [OPTION]... PATTERN [FILE]...
// options are pulled out by the command line processor
int fileOffset = 0;
if( StringUtils.isEmpty( regex ) ) {
regex = (String)argList.get( 0 );
fileOffset++;
}

// make sure its a good regex
try {
grepPattern = compilePattern( regex );
}
catch( final PatternSyntaxException pse ) {
printErrorMessage( "invalid regex syntax\nregex:" + regex + "\n" + pse.getMessage() );
return;
}

if( grepPattern == null ) {
return;
}

// just to get things going
processFileArgs( argList.subList( fileOffset, argList.size() ) );

// ok lets start
grepFiles();

}

private static void readExcludeFrom() {

String val = commandLine.getOptionValue( "exclude-from" );
if( StringUtils.isEmpty( val ) ) {
// trouble
return;
}

try( BufferedReader br = new BufferedReader( new FileReader( new File( val ) ) ) ) {
excludes = new ArrayList< String >();

for( String line = br.readLine(); line != null; line = br.readLine() ) {
excludes.add( line );
}

}
catch( IOException e ) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

private static long maxCount() {
if( commandLine.hasOption( 'm' ) || commandLine.hasOption( "max-count" ) ){
String s = commandLine.getOptionValue( 'm' );
return Long.parseLong( s );
}

return 0;
}

private static boolean printByteOffset() {

return commandLine.hasOption( 'b' ) || commandLine.hasOption( "byte-offset" );

}

public static void printErrorMessage( final String msg ) {
if( !( commandLine.hasOption( 's' ) || commandLine.hasOption( "no-messages" ) ) ) {
System.out.println(msg);
}
}

private static boolean printFileName() {

boolean foundH = commandLine.hasOption( 'H' ) || commandLine.hasOption( "with-filename" );
if( foundH ) {
return true;
}
boolean foundh = commandLine.hasOption( 'h' ) || commandLine.hasOption( "no-filename" );
if( foundh ) {
return false;
}

// what other factors?
return true;
}

private static boolean printFileNameOnly() {

return( commandLine.hasOption( 'l' ) || commandLine.hasOption( "files-with-matches" ) );
}

private static boolean printLineNumber() {

return( commandLine.hasOption( 'n' ) || commandLine.hasOption( "line-number" ) );
}

private static void printMatch( final File file, final String line, final long lineNumber,
final long count, final long byteOffset, final List< String > beforeContext, final List< String > afterContext ) {

if( quiet() || printFilesWithoutMatch() || printCountOnly() ) {
return;
}

/*
* if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
* System.out.println( "--" );
* }
*/

if( beforeContext() > 0 ) {
for( String bc : beforeContext ) {
String s = new String( bc );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

String msg = "";
if( printFileNamesOnly() ) {
try {
System.out.println( String.format( "%1$s", file.getCanonicalPath() ) );
}
catch( IOException ioe ) {
// TODO:
}
return;
}

if( printFileName() && ( filesToProcess.size() > 1 ) ) {
try {
msg += file.getCanonicalPath() + ":";
}
catch( IOException ioe ) {}
}

if( printLineNumber() ) {
msg += lineNumber + ":";
}

if( printByteOffset() ) {
msg += byteOffset + ":";
}

msg = msg.length() > 1 ? msg + " " + line : line;

System.out.println( msg );

if( afterContext() > 0 ) {
for( String ac : afterContext ) {
String s = new String( ac );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
System.out.println( "--" );
}
}

private static boolean printMatchOnly() {

return commandLine.hasOption( 'o' ) || commandLine.hasOption( "only-matching" );

}

// TODO: note that file names can contain wild cards too
// so this method should also expand directories as well
private static void processFileArgs( final List list ) {

filesToProcess = new ArrayList< File >();

for( final Object o : list ) {
final String name = (String)o;

final File f = new File( name );

if( f.exists() ) {
if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
filesToProcess.add( f );
}
}
}
else if( f.isDirectory() ) {
if( recurseDirectories() ) {
filesToProcess.addAll( recurseDir( f ) );
}
}
}
}

}

private static List< File > recurseDir( final File dir ) {

List< File > files = new ArrayList< File >();

for( File f : dir.listFiles() ) {

if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
files.add( f );
}
}
}
else if( f.isDirectory() ) {
if( !excludeDir( f ) ) {
files.addAll( recurseDir( f ) );
}
}

}

return files;

}

private static boolean recurseDirectories() {

if( commandLine.hasOption( 'd' ) || commandLine.hasOption( "directories" ) ) {

String val = commandLine.getOptionValue( 'd' );
if( StringUtils.endsWithIgnoreCase( val, "recurse" ) ) {
return true;
}

}

if( commandLine.hasOption( 'R' )
|| commandLine.hasOption( 'r' )
|| commandLine.hasOption( "recursive" ) ) {
return true;
}

return false;
}

private static boolean quiet() {

return commandLine.hasOption( 'q' ) || commandLine.hasOption( "--quiet" ) || commandLine.hasOption( "silent" );

}

private static String readRegexFromFile() {
String val = null;

String f = commandLine.getOptionValue( 'f' );
File file = new File(f);
if( file.exists() && file.isFile() ) {

try( BufferedReader br = new BufferedReader( new FileReader( f ) ) ){

String line = br.readLine();
if( StringUtils.isNotBlank( line ) ) {
val = line ;
}

}
catch( IOException e ) {
printErrorMessage( "Cannot open file:" + f );
}
}

return val ;
}

private static void reset() {

commandLine = null;
grepPattern = null;
regex = null;
filesToProcess = null;
excludes = null;


}

public static void usage() {

final String msg =
"Usage: grep [OPTION]... PATTERN [FILE]...\n" + "Search for PATTERN in each FILE or standard input.\n"
+ "PATTERN is, by default, a basic regular expression (BRE).\n"
+ "Example: grep -i 'hello world' menu.h main.c\n"
+ "Regexp selection and interpretation:\n"
+ " -E, --extended-regexp PATTERN is an extended regular expression (ERE)\n" // no
+ " -F, --fixed-strings PATTERN is a set of newline-separated fixed strings\n" // no
+ " -G, --basic-regexp PATTERN is a basic regular expression (BRE)\n" // no
+ " -P, --perl-regexp PATTERN is a Perl regular expression\n" // no
+ " -e, --regexp=PATTERN use PATTERN for matching\n" // done
+ " -f, --file=FILE obtain PATTERN from FILE\n" // done
+ " -i, --ignore-case ignore case distinctions\n" // done
+ " -w, --word-regexp force PATTERN to match only whole words\n" // done
+ " -x, --line-regexp force PATTERN to match only whole lines\n" // done
+ " -z, --null-data a data line ends in 0 byte, not newline\n" // yes?
+ "Miscellaneous:\n"
+ " -s, --no-messages suppress error messages\n" // done
+ " -v, --invert-match select non-matching lines\n" // done
+ " -V, --version print version information and exit\n" // done
+ " --help display this help and exit\n" // done
// +
+ " --mmap ignored for backwards compatibility\n" // no
+ "Output control:\n"
+ " -m, --max-count=NUM stop after NUM matches\n" // done - max-total
+ " -b, --byte-offset print the byte offset with output lines\n" // done
+ " -n, --line-number print line number with output lines\n" // done
+ " --line-buffered flush output on every line\n" // yes
+ " -H, --with-filename print the filename for each match\n" // done
+ " -h, --no-filename suppress the prefixing filename on output\n" // done
+ " --label=LABEL print LABEL as filename for standard input\n" // yes?
+ " -o, --only-matching show only the part of a line matching PATTERN\n" // done
+ " -q, --quiet, --silent suppress all normal output\n" // done
+ " --binary-files=TYPE assume that binary files are TYPE;\n" // yes?
+ " TYPE is `binary', `text', or `without-match'\n"
+ " -a, --text equivalent to --binary-files=text\n" // yes?
+ " -I equivalent to --binary-files=without-match\n" // yes?
+ " -d, --directories=ACTION how to handle directories;\n" // done
+ " ACTION is `read', `recurse', or `skip'\n"
+ " -D, --devices=ACTION how to handle devices, FIFOs and sockets;\n" // no
+ " ACTION is `read' or `skip'\n"
+ " -R, -r, --recursive equivalent to --directories=recurse\n" // done
+ " --include=FILE_PATTERN search only files that match FILE_PATTERN\n" // done
+ " --exclude=FILE_PATTERN skip files and directories matching FILE_PATTERN\n" // done
+ " --exclude-from=FILE skip files matching any file pattern from FILE\n" // done
+ " --exclude-dir=PATTERN directories that match PATTERN will be skipped.\n" // done
+ " -L, --files-without-match print only names of FILEs containing no match\n" // done
+ " -l, --files-with-matches print only names of FILEs containing matches\n" // done
+ " -c, --count print only a count of matching lines per FILE\n" // done
+ " -T, --initial-tab make tabs line up (if needed)\n" // no
+ " -Z, --null print 0 byte after FILE name\n" // no - use --sep=CHAR
+ "Context control:\n"
+ " -B, --before-context=NUM print NUM lines of leading context\n" // done
+ " -A, --after-context=NUM print NUM lines of trailing context\n" // done
+ " -C, --context=NUM print NUM lines of output context\n" // done
+ " -NUM same as --context=NUM\n" // no
+ " --color[=WHEN],\n" // no
+ " --colour[=WHEN] use markers to highlight the matching strings;\n" // no
+ " WHEN is `always', `never', or `auto'\n"
+ " -U, --binary do not strip CR characters at EOL (MSDOS)\n" // no?
+ " -u, --unix-byte-offsets report offsets as if CRs were not there (MSDOS)\n" // no?
+ "\n"
+ " `egrep' means `grep -E'. `fgrep' means `grep -F'.\n"
+ " Direct invocation as either `egrep' or `fgrep' is deprecated.\n"
+ " With no FILE, or when FILE is -, read standard input. If less than two FILEs\n"
+ " are given, assume -h. Exit status is 0 if any line was selected, 1 otherwise;\n"
+ " if any error occurs and -q was not given, the exit status is 2.\n"
+ "\n"
+ " Report bugs to: bug-grep@gnu.org\n"
+ " GNU Grep home page: \n"
+ " General help using GNU software: \n";
System.out.print( msg );

}

private static void version() {

System.out.println("Java Grep 0.1");

}

public static boolean includeFile( final File f ) {

if( commandLine.hasOption( "include" ) ) {

String pat = commandLine.getOptionValue( "include" );
String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, pat );

}

return true;
}

public static boolean excludeFile( final File f ) {

if( commandLine.hasOption( "exclude" ) ) {
String pat = commandLine.getOptionValue( "exclude" );
String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, pat );
}

if( commandLine.hasOption( "exclude-from" ) ) {
for( String pat : excludes ) {
String nm = FilenameUtils.getName( f.getName() );
boolean ex = FilenameUtils.wildcardMatch( nm, pat );
if( ex ) {
return true;
}
}
}

return false ;

}

private static boolean excludeDir( final File f ) {

if( commandLine.hasOption( "exclude-dir" ) ) {
String pat = commandLine.getOptionValue( "exclude-dir" );
if( FilenameUtils.wildcardMatch( FilenameUtils.getName( f.getName() ), pat ) ) {
return true;
}
}

return false;
}

private static boolean printFileNamesOnly() {

return printFilesWithoutMatch() || printFileNameOnly();
}

private static boolean printFilesWithoutMatch() {

return commandLine.hasOption( 'L' ) || commandLine.hasOption( "files-without-match" );
}

private static boolean printCountOnly() {

return commandLine.hasOption( 'c' ) || commandLine.hasOption( "count" );
}

private static long beforeContext() {

if( commandLine.hasOption( 'B' ) || commandLine.hasOption( "before-context" ) ) {
String s = commandLine.getOptionValue( 'B' );
return Long.parseLong( s );
}

return context();
}

private static long afterContext() {

if( commandLine.hasOption( 'A' ) || commandLine.hasOption( "after-context" ) ) {
String s = commandLine.getOptionValue( 'A' );
return Long.parseLong( s );
}

return context();
}

private static long context() {

if( commandLine.hasOption( 'C' ) || commandLine.hasOption( "context" ) ) {
String s = commandLine.getOptionValue( 'C' );
return Long.parseLong( s );
}

return 0;

}

private static void grepFile( final File file ) {

long maxCount = maxCount();
long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContext = new LinkedList< String >();
List< String > afterContext = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ), 16000 ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = grepPattern.matcher( line );
boolean found = m.find();

if( ( found && !invertMatch() ) ) {
count++;
if( afterContext() > 0 ) {
afterContext.clear();
bfr.mark( 4000 );
for( int i = 0; i < afterContext(); i++ ) { String s = bfr.readLine(); if( s != null ) { afterContext.add( s ); } else { break; } } bfr.reset(); } String match = line; if( printMatchOnly() ) { match = m.group(); } printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext ); if( printFileNameOnly() ) { return; } // } else if( !found && invertMatch() ) { count++; printMatch( file, line, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext ); // TODO: this has a slightly different meaning if( printFileNameOnly() ) { return; } } if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContext.add( line );
if( beforeContext.size() > beforeContext() ) {
beforeContext.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch() ) {
printMessage( file.getName() );
}
}

if( printCountOnly() ) {
printMessage( ( file.getName() + ":" + count ) );
}

}
}

Scala Collections Rant

Scala already has knocks against it for being confusing. So, I'll just kick it while it's down.

My latest frustration is with the mess that it calls Collections.

All I wanted to do was remove an item from a list. A very simple operation in just about every other programming environment ever created. I understand and appreciate that removing an item from an immutable collection should return a new list. In fact, that's exactly what I wanted to do.

So, I dug a little deeper in the Scala doc for List. But wait which List? Immutable List, which is just List. Mutable? Parrallel List? DoublyLinkedList? ListBuffer? Ok, this isn't so bad, I understand there are needs for all these.

So, lets just stick with Immutable List. Which is just called List. Even though Mutable List is called MutableList. Lets over look that too. For now.

OK. Open the Scala Doc for ImmutableList, I mean just List.

Ah, there's my remove method. But wait it's deprecated. Hmm. OK, maybe I'll use a MutableList. But wait, there's no remove method. What, no remove method on a MutableList? That's strange. Maybe its named something else? Nope. Maybe its an operator? Nope. I can contains, get, and indexOf on a MutableList. I can + an element onto a MutableList. But I can't - an element? I can also add items with + :+ +: +=: += . But not one operator to remove? What's up with that? Are these guys high on monads?

Ah, then there is a handy suggestion to use the filter function to remove the item I don't want. Good idea, except maybe I want to remove a specific item at a specific spot. Sounds bad doesn't it.

But all this functionality IS available. I was just looking at the wrong List. Its in ListBuffer. Whew. I guess I'll use a ListBuffer then. But now, I've just lost my the ImmutableLikeness of my List.

Or maybe I could have used an Immutable ListSet, just ListSet. But there is a possibility that I might want duplicates in the List. So I want a remove(index) function just like the measly old Java Util List. Which by the way must be jealous that it only has 20 methods vs 250 methods for the Scala List on steroids.

I could roll my own function. Or use some combination of take, head tail, etc. Or filter. But come on, its just remove.

So, that lead me to think about the API. List and MutableList I thought would basically have the same interface. Right? ... Well, right?

This discussion could get really long now.

A List is a LinearSeqOptimized, Product, LinearSeq, LinearSeq, LinearSeqLike, Seq, Seq, SeqLike, GenSeq, GenSeqLike, PartialFunction, Function1, Iterable, Iterable, IterableLike, Equals, GenIterable, GenIterableLike, Traversable, Immutable, Traversable, GenTraversable, GenericTraversableTemplate, TraversableLike, GenTraversableLike, Parallelizable, TraversableOnce, GenTraversableOnce, FilterMonadic, HasNewBuilder, AnyRef, Any.

And a MutableList is a Serializable, Serializable, Builder, Growable,
LinearSeqOptimized, LinearSeq, LinearSeq, LinearSeqLike, Seq, SeqLike, Cloneable, Seq, SeqLike, GenSeq, GenSeqLike, PartialFunction, Function1, Iterable, Iterable,
IterableLike, Equals, GenIterable, GenIterableLike, Traversable, Mutable, Traversable, GenTraversable, GenericTraversableTemplate, TraversableLike, GenTraversableLike, Parallelizable, TraversableOnce, GenTraversableOnce, FilterMonadic, HasNewBuilder, AnyRef, Any

That must explain the difference. Wait what.

Among 31 other things, a List is also a Function1? And a List is a HasNewBuilder? Isn't HasNewBuilder a question? Shouldn't that be NewBuilderLike? Maybe Buildable?

A MutableList is a Growable. But not a Shrinkable? Heresy.

And when did -like replace -able? Is this the new Valley Girl Moon Unit Zappa naming convention?

And it opens the possibility for the Likeable trait.

I really enjoy the name TraversableLike.

Why not use -ish? Then we can have TraverableLikeIsh.

Maybe throw in an -ly occasionally too?

So a List is TraverableOnce. Doesn't that mean the second time I try to loop over it. My code will fail?

Sausage, Politicians.

Now lets add Scala Collections.

What is Spring, really?

I should preface this with this statement.  I actually do like Spring.  At least Spring core.  And I use it almost every chance I get.

But I started to really think about it the other day.  What is Spring?  What is it doing?  What is dependency injection?

Here is a typical usage.  At least from my experience.

class SomeService {
    @Autowired
    SomeDao someDao;
}


So whats going on.  I'll tell you a dirty little secret.  SomeDao is a reference to a Global variable.  But here in OO land Global is an obscenity.

So what do we do to hide the fact that we are using Global variables but want to pretend we are OO kosher.

We use an annotation.  And 5 layers of javASSist.

What did this achieve?  This @Autowired.  Well it saved me from writing

SomeDao someDao = SomeDao.getInstance();

Which would probably be a Singleton pattern.  Which is evil too of course.

Or maybe I would have written it

SomeDao someDao = SomeDaoFactory.getSomeDao();

Factory how retro.  We don't use Factories any more.  They just pollute.

So, now I have a new clean elegant way to get my Globals.

I just use @Autowired

And that makes me feel a lot better.

Spring Guys - Enough with the names already

Can't you come up with a better naming scheme than AbstractThingThatDoesSomethingALittleDifferentFromThatOtherThingImpl

It makes my eyes bleed to read your Javadocs.

Oracle v Google

I think the the biggest question that no-one is asking is: Why didn't Google acquire Sun?

Hmmm

Any takers.

Everyone is whining about Oracle. But why didn't the geniuses at Google didn't figure it out?

Like every other tired American corp, Sun was tired of trying and begging for an acquisition. Pretty obvious to anyone watching quarterly reports.

IBM wasn't going to do it. Too Indian. Microsoft. Come on. That pretty much left Oracle.

So wheres the surprise?

Apache Leaving JCP Side Note

Take a breath. Good. In... Out...

OK.

Now stop and think.

This is not really that bad for Java the language. Or Java the community. And now lets hope Apache goes forward with Harmony and creates something special. If they put a different language on it, even better.

Does the community need JSRs? Does it really need "standards"? (Yep I quoted that. We could do a whole nother post on Sun/Oracle specs parading as standards. Or how if you wait 10 years for ISO standards you wind up with C++.)

No and No.

Java in the last 7 or so years has been dominated by projects that are not standards. Why? Because they are solid, dependable, growing projects. Lets name a few. Log4j/slf4j. Struts. Hibernate. Spring. That took 5 seconds. And if I spent the time, I could easily rattle off a dozen more.

Plain and simple, these projects fill needs that software developers have. If the Java community sat around and waited for Sun (or Oracle) to fill every hole, then the Java community really would have died on the vine.

In fact waiting for a large company to solve basic issues is a crap shoot at best. Sun did the world a huge favor releasing Java and the JVM. They spent millions of dollars developing Java/JVM technology. And basically gave it away for free. For that, they deserve a tremendous amount of thanks.

However, for 15+ years, they failed to really nurture it in some aspects. I would argue that is because, they didn't know what to do with it or how to capitalize on it.

For example, 15+ years later and we have as part of the standard
poor date handling
poor math libs
poor collections libs
poor ORM (JPA is a disaster. Like JDO and EJB/EJBQL before it.)
poor CDI
two weak tries at web ui interaction (JSP & JSF. Can we please stick the fork in JSF?)
poor threading/concurrency model (Its getting better.)

Projects and people outside Sun did a fantastic job of rising to the occasion and filling these needs. Without standards without JSRs and without JCPs.

Scala Cons

Here is one of my biggest gripes with Scala.
It has a very irregular syntax.

Here are two examples straight from Odersky's book.


for (p <- persons; n = p.name; if (n startsWith "To")
yield n

for {

        p <- persons
        // a generator
        n = p.name // a definition
        if (n startsWith "To") // a filter

} yield n


These are equivalent for expressions.  In my opinion there is a huge problem with this.

Sometimes I can do for() sometimes for{} and they are the same.
I think this reflects poor design.  Someone thought they were clever and got the grammar to do the same thing so they left it.

But syntax, including punctuation, is very important in how quickly someone can read text and comprehend it.

Now, if someone is reading Scala code, they have to be very careful about how they mentally parse and interpret a block of code.

These are valuable mental cycles that must to be spent on the syntax of the language, instead of the semantics of the code.

When looking at Java or C for example, a developer doesn't have this issue.
A for loop always uses parens, period end of story.  There is no time spent thinking about paren vs brace.  Or semi colon vs missing semi colon.

Scala Pros & Cons

Here's a quick list of things that Scala does right and wrong.

And I'm just making this list for myself.

Pros
1. Brings together the OO and Functional realms.  Sure other languages have done it before.  But probably not as well and seamlessly as Scala.
2. Actors.  Great concept.  Also, not original.  But it works and is easy to implement.
3. Operator overloading.  This is definitely one this that makes code read easier to read.
var thirdList = firstList + secondList ;
How hard is that Java guys?
List thirdList = new List( firstList );
thirdList.addAll( secondList );

4. Traits.  Seems pretty simple after you use it once or twice.  Makes you wonder what all the fuss was about in Java banning multiple inheritance.

5. The type system.  In some cases it makes life a lot easier.
I can pass in a List of Dogs into a function that takes a List of Mammals.  Just seems natural.
More on the type system below.

6. Runs on the JVM.  I almost didn't mention it.  I don't think its really that big of a deal.  People have been able to solve all kinds of problems with plain old java.  Scala needs to give the world a compelling reason to not use java.

I'm almost reminded here of the days when o/s 2 ran windows apps.  Super, if I can develop apps that run on windows, why should I bother writing apps specifically for o/s 2?  And hence everyone sticks with windows.  Its not quite an analogy, just a similarity.

Cons
1. Irregular syntax.  And this is a biggie.  A regular syntax makes code easy to read and pick up from project to project.  The philosophy of flexible syntax is nice, but in practice its a bad idea.  In general, I'm not a fan of lisp.  However, the simplicity and regularity of lisp syntax is one of the major reasons its hung around for so long.

If I pick up a new piece of code that I am supposed to maintain or debug, I don't want to follow the guy who thought
x | y -> z -> a
is easy to understand.  Was this guy a Haskell coder?  Which one is the object? which one is the function?  What does | do again?  And why the heck is anyone in this day and age using one character variable names?  Unless this is 3d geometry.

2. The type system.  Sure its powerful.  But I cant really think of too many times in the last twenty years where I needed a full blown type calculus system to solve a problem.  And this seems to be one area where people like to gravitate towards when they take a shot as Scala.

Thats it for now.  More to come.  Notice there are only two Cons.  Yes, I'm a fan of the language.

One last go around

grammar SXML ;

options{ k=4; backtrack=true; memoize=true; }

import XMLLexer ;

xmlDocument
    : prolog element misc*
    ;

/*
prolog
    : misc* docType?  misc*
    ;   
*/   
   
prolog
    : misc*
    ;       
   
misc
    : comment
    | pi
    | pcData
    | docType
    | WS
    ;

// simple items   
comment : COMMENT ;
pi      : PI {System.out.println("found pi");};
cdata   : CDATA ;       
pcData  : PCDATA ;

docType
    : DOCTYPE_OPEN (WS Name)+ (WS ATTR_VAL)* WS? docTypeDef? WS? eoe=ELE_OPEN_END
        {System.out.println("found doctype:"+ $eoe );}
    ;
   
docTypeDef
    :    LBRACKET ( docTypeDefItem )* RBRACKET
    ;
   
docTypeDefItem
    : elementDecl
    | attListDecl
    | entityDecl
    | notationDecl
    | pi
    | comment         
    | WS
    | pcData
    ;

elementDecl
    : ELEMENT_OPEN WS Name WS elementDeclContent WS? ELE_OPEN_END
    ;
   
elementDeclContent
    : Name
    | mixedContent
    | childrenContent
    ;
   
mixedContent
    : LPAREN WS? mixedContentItem ( WS? PIPE WS? mixedContentItem )* WS? RPAREN childOper?
    ;
   
mixedContentItem
    : POUND Name
    | Name
    | PCT Name SEMI
    ;               
   
childrenContent
    : (choice | sequence ) childOper?
    ;   
   
choice
    : LPAREN WS? child ( WS? PIPE WS? child )+  WS? RPAREN
    ;   
   
sequence
    : LPAREN WS? child ( WS? COMMA WS? child )*  WS? RPAREN
    ;       
   
child
    : Name childOper?
    | PCT Name SEMI childOper?
    | choice childOper?
    | sequence childOper?
    ;   

childOper
    :    QUEST | STAR | PLUS
    ;
       

   
attListDecl
    : ATTLIST_OPEN WS Name (attListDef)* WS? ELE_OPEN_END
    ;
   
attListDef
    : WS Name WS attListType WS defaultDecl
    ;       
   
attListType
    : Name
    | Name WS LPAREN WS? Name (WS? PIPE WS? Name)* WS? RPAREN
    | LPAREN WS? Name (WS? PIPE WS? Name)* WS? RPAREN
    ;
   
defaultDecl
    : ATTR_VAL
    | POUND Name (WS ATTR_VAL)?
    ;
        
   
entityDecl
    : ENTITY_OPEN (WS PCT)? (WS Name)+ (WS ATTR_VAL)* (WS Name)* WS? ELE_OPEN_END
    ;
   
notationDecl
    : NOTATION_OPEN WS Name (ATTR_VAL WS)* WS? ELE_OPEN_END
    ;       


element : emptyElement
        | contentElement
        ;
       
contentElement
    : ELE_OPEN_START Name (WS (attribute WS?)* )? ELE_OPEN_END
        content
        ELE_CLOSE_START Name ELE_OPEN_END
    ;
   
emptyElement : ELE_OPEN_START Name (WS (attribute WS?)* )? ELE_EMPTY_END ;
       
attribute : Name WS? ATTR_EQ WS? ATTR_VAL ;       
   
       
content   
    : contentItem*
    ;
   
contentItem
    : element
    | cdata 
    | pi
    | comment
    | pcData
    | WS
    ;   


lexer grammar XMLLexer ;

options{ k=11; backtrack=true; }

// memoize=true; }

@members {

    boolean inTag     = false;
    boolean inDocType = false;
    boolean inDocEle  = false;
   
    public boolean inEle(){
        return inDocType || inDocEle || inTag ;
    }
   
    public void closeDocType(){
        if( inDocEle ){
            inDocEle = false ;
            return;
        }
       
        if( inDocType ){
            inDocType = false ;
            return ;
        }
    }

}



// processing instructions
PI : PI_OPEN ( options{greedy=false;}: . )* PI_CLOSE ;
PI_OPEN : 'PI_CLOSE : '?>' ;

// comments
COMMENT : COMMENT_OPEN ( options{greedy=false;}: . )* COMMENT_CLOSE ;
COMMENT_OPEN : '' ;

// cdata
CDATA : CDATA_OPEN ( options{greedy=false;} : . )*  CDATA_CLOSE ;
CDATA_OPEN : '' ;

// includes
INCLUDE : INCLUDE_OPEN ( options{greedy=false;} : . )*  CDATA_CLOSE ;
INCLUDE_OPEN : '
CARET : '^' ;   
   
// 005F   
UNDER_SCORE : '_' ;   

BACK_TICK :    '`' ;

//     0061..007A
LOWER : 'a'..'z' ;

// 007D
PIPE : '|' ;

// 007E
TILDE : '~' ;

LCURLY : '{' ;
RCURLY : '}' ;

UNICODE_MISC
    : '\u00A0'..'\uD7FF'
    | '\uE000'..'\uFFFD' ;
   
// [3]
// #x20 ' ' \u0020
// #xA '\n'  \u000A
// #xD '\r'  \u000D
// #x9 '\t' \u0009
// do not preserve whitespace {$channel=HIDDEN;}
WS
    : ( '\r' | '\n' | '\t' | ' ' )+
    ;
   
SINK
    : sc=. { System.out.println("found sink.  char:"+ $sc ); };   

One Antlr grammar

I broke out the parser and the lexer because it was a pain to lex the xml and embedded doc types at the same time.  Only issue now is UTF-16.  Which may or may not be an antlr short coming.  Last post on Antlr.  Next time I'll rant on the iPhone and Android.

Here's the simplified parser and lexer.  And it really is much much simpler.  I'm thinking that if I really want to validate the XML, I'll add second pass parsing on an embedded doctype.  And entity expansion.  But for now this parses everything I need.  And the combined lexer/parser comes in at 100K vs. 2MB for Xerces.

I'm sure I can speed up performance tweaking k=x, backtrack, and memoize.  But its fine for now.

grammar SXML ;

options{ k=4; backtrack=true; memoize=true; }

/*
    this version has issues with quotes in element data
   
    starting an effort to move more logic down to the lexer
   
    and adding perserve whitespace lexwing / parsing
*/
import XMLLexer ;

// 1
xmlDocument
    : prolog element misc*
    ;

   
prolog
    : misc* doctypedecl?  misc*
    ;   
   
// [27]
misc
    : textType
    | comment
    | pi
    | WS
    ;
   
textType
    : ( UNICODE_MISC )+ { System.out.println( "found textType" ); }
    ;       

// 2.5 Comments
// [15] - should a comment be a parser rule?
comment
    : COMMENT
    ;
   
   
// 2.8   
// Document Type Definition
// [28]
doctypedecl : DOCTYPE ;


// 2.6 Processing Instructions
// [16]
pi
    : PI
    ;
   

// 3 .0
// Element
// [39]
element : ELE_EMPTY
        | ELE_OPEN content ELE_CLOSE
        ;
       
cdata
    :    CDATA ;       


// Content of Elements
// [43]
content   
    : contentItem*
    ;
   
contentItem
    : element
    | cdata 
    | pi
    | comment
    | WS
    | String
    | singleChar
    | COMMENT_CLOSE
    | PI_CLOSE
    | CDATA_CLOSE   
    ;   
   
singleChar
    : BANG
    | DQUOTE
    | POUND
    | DOLLAR
    | PCT
    | AMP
    | SQUOTE
    | LPAREN
    | RPAREN
    | STAR
    | PLUS
    | COMMA
    | MINUS
    | MINUSMINUS
    | DOT
    | FWD_SLASH
    | DIGIT
    | COLON
    | SEMI
    | LT
    | EQ
    | GT
    | QUEST
    | AT
    | UPPER
    | LBRACKET
    | BACK_SLASH
    | RBRACKET
    | CARET
    | UNDER_SCORE
    | BACK_TICK
    | LOWER
    | PIPE
    | TILDE
    | LCURLY
    | RCURLY
    | UNICODE_MISC
    | SINK
    ;   

lexer grammar XMLLexer ;

options{ k=4; backtrack=true; memoize=true; }

/*
    A lexer to create tokens for the xml grammar
*/


PI : PI_OPEN ( options{greedy=false;}: . )* PI_CLOSE ;
PI_OPEN : 'PI_CLOSE : '?>' ;

COMMENT : COMMENT_OPEN ( options{greedy=false;}: . )* COMMENT_CLOSE ;
COMMENT_OPEN : '' ;

CDATA : CDATA_OPEN ( options{greedy=false;} : . )*  CDATA_CLOSE ;
fragment CDATA_OPEN : '' ;

INCLUDE : INCLUDE_OPEN ( options{greedy=false;} : . )*  CDATA_CLOSE ;
INCLUDE_OPEN : '
CARET : '^' ;   
   
// 005F   
UNDER_SCORE : '_' ;   

BACK_TICK :    '`' ;

//     0061..007A
LOWER : 'a'..'z' ;

// 007D
PIPE : '|' ;

// 007E
TILDE : '~' ;

LCURLY : '{' ;
RCURLY : '}' ;

UNICODE_MISC
    : '\u00A0'..'\uD7FF'
    | '\uE000'..'\uFFFD' ;
   
// [3]
// #x20 ' ' \u0020
// #xA '\n'  \u000A
// #xD '\r'  \u000D
// #x9 '\t' \u0009
// do not preserve whitespace {$channel=HIDDEN;}
WS
    : ( '\r' | '\n' | '\t' | ' ' )+
    ;
   
SINK
    : . { System.out.println("found sink"); };