Monday, November 21, 2011

Grep some more

Here is an update to the Java grep. I am doing this as I follow the Mastering Regular Expressions book to that I can test the java examples. And hopefully make the Java examples less awkward.

its a little cleaner than the last one.

package org.rip.regex;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;

public class grep {

public grep() {

grepPatterns = new ArrayList< Pattern >();
filesToProcess = new ArrayList< File >();
excludes = new ArrayList< String >();
regexes = new ArrayList< String >();
}

private List< Pattern > grepPatterns;

private List< String > regexes;

private List< File > filesToProcess;

private ArrayList< String > excludes;

public Pattern compilePattern( final String pat ) {

int flags = 0;

String pattern = new String( pat );

if( ignoreCase ) {
flags |= Pattern.CASE_INSENSITIVE;
}

// we are ignoring line-regex if both are supplied
if( wordRegex ) {
// poor mans way to do it
pattern = "\\b" + pattern + "\\b";
}
else {
if( lineRegex ) {

// poor mans way to do it
pattern = "^" + pattern + "$";
}
}

return Pattern.compile( pattern, flags );

}


private void printMessage( final String msg ) {

if( !quiet ) {
System.out.println( msg );
}
}

private void grepFiles() {

// at this point the list was expanded into file names
for( final File f : filesToProcess ) {

grepFile( f );
}

}

private void transferOptions( final CommandLine commandLine ) {

long context = 0;

if( commandLine.hasOption( 'C' ) || commandLine.hasOption( "context" ) ) {
String s = commandLine.getOptionValue( 'C' );
context = Long.parseLong( s );
}


if( commandLine.hasOption( 'A' ) || commandLine.hasOption( "after-context" ) ) {
String s = commandLine.getOptionValue( 'A' );
afterContext = Long.parseLong( s );
}
else {
afterContext = context;
}

if( commandLine.hasOption( 'B' ) || commandLine.hasOption( "before-context" ) ) {
String s = commandLine.getOptionValue( 'B' );
beforeContext = Long.parseLong( s );
}
else {
beforeContext = context;
}

if( commandLine.hasOption( 'i' ) || commandLine.hasOption( "ignore-case" ) ) {
ignoreCase = true;
}

if( commandLine.hasOption( 'v' ) || commandLine.hasOption( "invert-match" ) ) {
invertMatch = true;
}

maxCount = 0;
if( commandLine.hasOption( 'm' ) || commandLine.hasOption( "max-count" ) ) {
String s = commandLine.getOptionValue( 'm' );
maxCount = Long.parseLong( s );
}

if( commandLine.hasOption( 'l' ) || commandLine.hasOption( "files-with-matches" ) ) {
printFileNameOnly = true;
}

if( commandLine.hasOption( 'b' ) || commandLine.hasOption( "byte-offset" ) ) {
printByteOffset = true;
}

if( commandLine.hasOption( 'q' ) || commandLine.hasOption( "--quiet" ) || commandLine.hasOption( "silent" ) ) {
quiet = true;
}

if( commandLine.hasOption( 'c' ) || commandLine.hasOption( "count" ) ) {
printCountOnly = true;
}

printFilesWithoutMatch = commandLine.hasOption( 'L' ) || commandLine.hasOption( "files-without-match" );

wordRegex = commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" );

lineRegex = commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" );

if( commandLine.hasOption( "exclude-from" ) ) {
readExcludeFrom( commandLine.getOptionValue( "exclude-from" ) );
}

// this could be specified multiple times
if( commandLine.hasOption( 'e' ) || commandLine.hasOption( "regexp" ) ) {
regexes.add( commandLine.getOptionValue( 'e' ) );
}

if( commandLine.hasOption( 'f' ) || commandLine.hasOption( "file" ) ) {
readRegexFromFile( commandLine.getOptionValue( 'f' ) );
}

noMessages = commandLine.hasOption( 's' ) || commandLine.hasOption( "no-messages" );

if( commandLine.hasOption( 'H' ) || commandLine.hasOption( "with-filename" ) ) {
printFileName = true;
}

if( commandLine.hasOption( 'h' ) || commandLine.hasOption( "no-filename" ) ) {
printFileName = false;
}

printMatchOnly = commandLine.hasOption( 'o' ) || commandLine.hasOption( "only-matching" );

printLineNumber = commandLine.hasOption( 'n' ) || commandLine.hasOption( "line-number" );

recurseDirectories = false;

if( commandLine.hasOption( 'd' ) || commandLine.hasOption( "directories" ) ) {

String val = commandLine.getOptionValue( 'd' );
if( StringUtils.endsWithIgnoreCase( val, "recurse" ) ) {
recurseDirectories = true;
}

}

if( commandLine.hasOption( 'R' )
|| commandLine.hasOption( 'r' )
|| commandLine.hasOption( "recursive" ) ) {
recurseDirectories = true;
}

if( commandLine.hasOption( "include" ) ) {
includeFilePattern = commandLine.getOptionValue( "include" );
}

if( commandLine.hasOption( "exclude" ) ) {
excludeFilePattern = commandLine.getOptionValue( "exclude" );
}

if( commandLine.hasOption( "exclude-dir" ) ) {
excludeDirPattern = commandLine.getOptionValue( "exclude-dir" );
}

}

private boolean invertMatch = false;
private boolean ignoreCase = false;
private long maxCount;

private boolean printFileNameOnly = false;

private boolean printByteOffset = false;

private boolean quiet = false;

private boolean printCountOnly = false;

private boolean printFilesWithoutMatch = false;

private boolean wordRegex = false;

private boolean lineRegex = false;

private boolean noMessages = false;

private boolean printFileName = true;

private boolean printMatchOnly = false;
private boolean printLineNumber=false;

private boolean recurseDirectories = false;

private String includeFilePattern = null;

private String excludeFilePattern = null;

private String excludeDirPattern = null;

public static void main( final String[] args ) {

grep theGrep = new grep();
grepCommandLine cmd = new grepCommandLine();

if( ArrayUtils.isEmpty( args ) ) {
grepCommandLine.usage();
return;
}

// parse the command line
try {
final CommandLineParser parser = new PosixParser();
cmd.setCommandLine( parser.parse( grepCommandLine.grepOptions, args ) );
}
catch( final ParseException pe ) {
System.out.print( pe.getMessage() );
grepCommandLine.usage();
return;
}

theGrep.transferOptions( cmd.getCommandLine() );

// if version or help was given then print and quit
if( cmd.getCommandLine().hasOption( "help" ) ) {
grepCommandLine.usage();
return;
}

if( cmd.getCommandLine().hasOption( "version" ) || cmd.getCommandLine().hasOption( 'V' ) ) {
grepCommandLine.version();
return;
}

//
if( cmd.getCommandLine().hasOption( 'e' ) || cmd.getCommandLine().hasOption( "regexp" ) ) {
if( ArrayUtils.isEmpty( cmd.getCommandLine().getOptionValues( 'e' ) ) ) {
grepCommandLine.usage();
return;
}
}

if( cmd.getCommandLine().hasOption( 'f' ) || cmd.getCommandLine().hasOption( "file" ) ) {
if( ArrayUtils.isEmpty( cmd.getCommandLine().getOptionValues( 'f' ) ) ) {
grepCommandLine.usage();
return;
}
}

if( cmd.getCommandLine().hasOption( "exclude-from" ) ) {
String s = cmd.getCommandLine().getOptionValue( "exclude-from" );
if( StringUtils.isEmpty( s ) ) {
grepCommandLine.usage();
return;
}
}

// get the unprocessed items
final List argList = cmd.getCommandLine().getArgList();

// we must have some args besides options
if( CollectionUtils.isEmpty( argList ) ) {
// TODO - regex could be passed from file or arg
// input could be from STDIN
grepCommandLine.usage();
return;
}

// just to get things started
// we are assuming: [OPTION]... PATTERN [FILE]...
// options are pulled out by the command line processor
// if the regex was not supplied via file or arg
// we assume its the first arg
// if the regex was set via arg or file
// we assume the first arg is a file
int fileOffset = 0;
if( CollectionUtils.isEmpty( theGrep.regexes ) ) {
theGrep.regexes.add( (String)argList.get( 0 ) );
fileOffset++;
}

// make sure its a good regex
try {
theGrep.compilePatterns();
}
catch( final PatternSyntaxException pse ) {
theGrep.printErrorMessage( "invalid regex syntax\n" + pse.getMessage() );
return;
}

if( CollectionUtils.isEmpty( theGrep.grepPatterns ) ) {
grepCommandLine.usage();
return;
}

// just to get things going
List< String > files = argList.subList( fileOffset, argList.size() );
if( files.size() > 0 ) {
theGrep.printFileName = true;
}
theGrep.processFileArgs( files );

// ok lets start
theGrep.grepFiles();

}

private void compilePatterns() {

for( String s : regexes ) {
grepPatterns.add( compilePattern( s ) );
}
}

private void readExcludeFrom( final String val ) {


try( BufferedReader br = new BufferedReader( new FileReader( new File( val ) ) ) ) {
excludes = new ArrayList< String >();

for( String line = br.readLine(); line != null; line = br.readLine() ) {
excludes.add( line );
}

}
catch( IOException e ) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}





public void printErrorMessage( final String msg ) {

if( !noMessages ) {
System.out.println(msg);
}
}







private void printMatch( final File file, final String line, final long lineNumber,
final long count, final long byteOffset, final List< String > beforeContextLines,
final List< String > afterContextLines ) {

if( quiet || printFilesWithoutMatch || printCountOnly ) {
return;
}

/*
* if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
* System.out.println( "--" );
* }
*/

if( beforeContext > 0 ) {
for( String bc : beforeContextLines ) {
String s = new String( bc );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

String msg = "";
if( printFileNamesOnly() ) {
try {
System.out.println( String.format( "%1$s", file.getCanonicalPath() ) );
}
catch( IOException ioe ) {
// TODO:
}
return;
}

if( printFileName ) {
try {
msg += file.getCanonicalPath() + ":";
}
catch( IOException ioe ) {}
}

if( printLineNumber ) {
msg += lineNumber + ":";
}

if( printByteOffset ) {
msg += byteOffset + ":";
}

msg = msg.length() > 1 ? msg + " " + line : line;

System.out.println( msg );

if( afterContext > 0 ) {
for( String ac : afterContextLines ) {
String s = new String( ac );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

if( ( beforeContext > 0 ) || ( afterContext > 0 ) ) {
System.out.println( "--" );
}
}



// TODO: note that file names can contain wild cards too
// so this method should also expand directories as well
private void processFileArgs( final List list ) {

filesToProcess = new ArrayList< File >();

for( final Object o : list ) {
final String name = (String)o;

final File f = new File( name );

if( f.exists() ) {
if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
filesToProcess.add( f );
}
}
}
else if( f.isDirectory() ) {
if( recurseDirectories ) {
filesToProcess.addAll( recurseDir( f ) );
}
}
}
}

}

private List< File > recurseDir( final File dir ) {

List< File > files = new ArrayList< File >();

for( File f : dir.listFiles() ) {

if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
files.add( f );
}
}
}
else if( f.isDirectory() ) {
if( !excludeDir( f ) ) {
files.addAll( recurseDir( f ) );
}
}

}

return files;

}





private void readRegexFromFile( final String fname ) {


File file = new File( fname );
if( file.exists() && file.isFile() ) {

try( BufferedReader br = new BufferedReader( new FileReader( file ) ) ) {

String line = br.readLine();
if( StringUtils.isNotBlank( line ) ) {
regexes.add( line );
}

}
catch( IOException e ) {
printErrorMessage( "Cannot open file:" + fname );
}
}


}

private void reset() {

grepPatterns = new ArrayList< Pattern >();
regexes = new ArrayList< String >();
filesToProcess = null;
excludes = null;


}





public boolean includeFile( final File f ) {

if( StringUtils.isNotEmpty( includeFilePattern ) ) {

String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, includeFilePattern );

}

return true;
}

public boolean excludeFile( final File f ) {

if( StringUtils.isNotBlank( excludeFilePattern ) ) {

String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, excludeFilePattern );
}

if( CollectionUtils.isNotEmpty( excludes ) ) {
for( String pat : excludes ) {
String nm = FilenameUtils.getName( f.getName() );
boolean ex = FilenameUtils.wildcardMatch( nm, pat );
if( ex ) {
return true;
}
}
}

return false ;

}

private boolean excludeDir( final File f ) {

if( StringUtils.isNotBlank( excludeDirPattern ) ) {
if( FilenameUtils.wildcardMatch( FilenameUtils.getName( f.getName() ), excludeDirPattern ) ) {
return true;
}
}

return false;
}

private boolean printFileNamesOnly() {

return printFilesWithoutMatch || printFileNameOnly;
}




private long beforeContext = 0;

private long afterContext = 0;

private Matcher matchAny( final String line ) {

for( Pattern pat : grepPatterns ) {

Matcher m = pat.matcher( line );
if( m.find() ) {
return m;
}
}

return null;
}

private void grepFile( final File file ) {


long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContextLines = new LinkedList< String >();
List< String > afterContextLines = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ), 16000 ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = matchAny( line );

if( ( ( m != null ) && !invertMatch ) ) {
count++;
if( afterContext > 0 ) {
afterContextLines.clear();
bfr.mark( 4000 );
for( int i = 0; i < afterContext; i++ ) { String s = bfr.readLine(); if( s != null ) { afterContextLines.add( s ); } else { break; } } bfr.reset(); } String match = line; if( printMatchOnly ) { match = m.group(); } printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContextLines, afterContextLines ); if( printFileNameOnly ) { return; } // } else if( ( m == null ) && invertMatch ) { count++; printMatch( file, line, lineNumber, count, byteOffset, beforeContextLines, afterContextLines ); // TODO: this has a slightly different meaning if( printFileNameOnly ) { return; } } if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContextLines.add( line );
if( beforeContextLines.size() > beforeContext ) {
beforeContextLines.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch ) {
printMessage( file.getName() );
}
}

if( printCountOnly ) {
printMessage( ( file.getName() + ":" + count ) );
}

}

private boolean isRegexSet() {

return CollectionUtils.isNotEmpty( regexes );
}
}



package org.rip.regex;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;

public class grepCommandLine {

static Options grepOptions;

static {
grepCommandLine.grepOptions = createOptions();
}

private CommandLine commandLine;

public grepCommandLine() {

}

public CommandLine getCommandLine() {

return commandLine;
}

public void setCommandLine( final CommandLine commandLine ) {

this.commandLine = commandLine;
}

private static Options createOptions() {

final Options opts = new Options();

opts.addOption( "E", "extended-regexp", false, "PATTERN is an extended regular expression (ERE)" );
opts.addOption( "F", "fixed-strings", false, "PATTERN is a set of newline-separated fixed strings" );
opts.addOption( "G", "basic-regexp", false, "PATTERN is a basic regular expression (BRE)" );
opts.addOption( "P", "perl-regexp", false, "PATTERN is a Perl regular expression" );
opts.addOption( OptionBuilder.withLongOpt( "regexp" ).hasArg().withArgName( "PATTERN" )
.withDescription( "use PATTERN for matching" ).create( 'e' ) );
opts.addOption( OptionBuilder.withLongOpt( "file" ).hasArg().withArgName( "FILE" )
.withDescription( "obtain PATTERN from FILE" ).create( 'f' ) );
opts.addOption( "i", "ignore-case", false, "ignore case distinctions" );
opts.addOption( "w", "word-regexp", false, "force PATTERN to match only whole words" );
opts.addOption( "x", "line-regexp", false, "force PATTERN to match only whole lines" );
opts.addOption( "z", "null-data", false, "a data line ends in 0 byte, not newline" );
opts.addOption( "s", "no-messages", false, "suppress error messages" );
opts.addOption( "v", "invert-match", false, "select non-matching lines" );
opts.addOption( "V", "version", false, "print version information and exit" );
opts.addOption( OptionBuilder.withLongOpt( "help" ).withDescription( "display this help and exit" ).create() );
// opts.addOption( OptionBuilder.withLongOpt( "mmap" ).withDescription(
// "ignored for backwards compatibility" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "max-count" ).withArgName( "NUM" ).hasArg()
.withDescription( "stop after NUM matches" ).create( 'm' ) );
opts.addOption( "b", "byte-offset", false, "print the byte offset with output lines" );
opts.addOption( "n", "line-number", false, "print line number with output lines" );
opts.addOption( OptionBuilder.withLongOpt( "line-buffered" ).withDescription( "flush output on every line" )
.create() );
opts.addOption( "H", "with-filename", false, "print the filename for each match" );
opts.addOption( "h", "no-filename", false, "suppress the prefixing filename on output" );
opts.addOption( OptionBuilder.withLongOpt( "label" ).hasArg().withArgName( "LABEL" )
.withDescription( "print LABEL as filename for standard input" ).create() );
opts.addOption( "o", "only-matching", false, "show only the part of a line matching PATTERN" );
opts.addOption( "q", "quiet", false, "suppress all normal output" );
opts.addOption( OptionBuilder.withLongOpt( "silent" ).withDescription( "suppress all normal output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "binary-files" ).hasArg().withArgName( "TYPE" )
.withDescription( "assume that binary files are TYPE" ).create() );
// TYPE is `binary', `text', or `without-match'
opts.addOption( "a", "text", false, "equivalent to --binary-files=text" );
opts.addOption( "I", false, "equivalent to --binary-files=without-match" );
opts.addOption( OptionBuilder.withLongOpt( "directories" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle directories" ).create( 'd' ) );
// ACTION is `read', `recurse', or `skip'"
opts.addOption( OptionBuilder.withLongOpt( "devices" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle devices, FIFOs and sockets" ).create( 'D' ) );
// ACTION is `read' or `skip'
opts.addOption( "R", false, "equivalent to --directories=recurse" );
opts.addOption( "r", "recursive", false, "equivalent to --directories=recurse" );
opts.addOption( OptionBuilder.withLongOpt( "include" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "search only files that match FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "skip files and directories matching FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-from" ).hasArg().withArgName( "FILE" )
.withDescription( "skip files matching any file pattern from FILE" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-dir" ).hasArg().withArgName( "PATTERN" )
.withDescription( "directories that match PATTERN will be skipped" ).create() );
opts.addOption( "L", "files-without-match", false, "print only names of FILEs containing no match" );
opts.addOption( "l", "files-with-matches", false, "print only names of FILEs containing matches" );
opts.addOption( "c", "count", false, "print only a count of matching lines per FILE" );
opts.addOption( "T", "initial-tab", false, "make tabs line up (if needed)" );
opts.addOption( "Z", "null", false, "print 0 byte after FILE name" );
opts.addOption( OptionBuilder.withLongOpt( "before-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of leading context" ).create( 'B' ) );
opts.addOption( OptionBuilder.withLongOpt( "after-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of trailing context" ).create( 'A' ) );
opts.addOption( OptionBuilder.withLongOpt( "contex" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of output context" ).create( 'C' ) );
// * + " -NUM same as --context=NUM\n" +
opts.addOption( OptionBuilder.withLongOpt( "color" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "colour" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
// *
// " WHEN is `always', `never', or `auto'\n"
opts.addOption( "U", "binary", false, "do not strip CR characters at EOL (MSDOS)" );
opts.addOption( "u", "unix-byte-offsets", false, "report offsets as if CRs were not there (MSDOS)" );

// non gnu options
opts.addOption( OptionBuilder.withLongOpt( "debug" ).withDescription( "turn on debugging output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "verbose" ).withDescription( "turn on verbose output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "test" ).withDescription( "run in test mode" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "logging" ).withDescription( "use nice logging, Log4J/SLF4J" )
.create() );
opts.addOption( OptionBuilder.withLongOpt( "file-long" ).hasArg().withArgName( "FILE" )
.withDescription( "read PATTERN from file using long format" ).create() );

return opts;
}

public static void usage() {

final String msg =
"Usage: grep [OPTION]... PATTERN [FILE]...\n"
+ "Search for PATTERN in each FILE or standard input.\n"
+ "PATTERN is, by default, a basic regular expression (BRE).\n"
+ "Example: grep -i 'hello world' menu.h main.c\n"
+ "Regexp selection and interpretation:\n"
+ " -E, --extended-regexp PATTERN is an extended regular expression (ERE)\n" // no
+ " -F, --fixed-strings PATTERN is a set of newline-separated fixed strings\n" // no
+ " -G, --basic-regexp PATTERN is a basic regular expression (BRE)\n" // no
+ " -P, --perl-regexp PATTERN is a Perl regular expression\n" // no
+ " -e, --regexp=PATTERN use PATTERN for matching\n" // done
+ " -f, --file=FILE obtain PATTERN from FILE\n" // done
+ " -i, --ignore-case ignore case distinctions\n" // done
+ " -w, --word-regexp force PATTERN to match only whole words\n" // done
+ " -x, --line-regexp force PATTERN to match only whole lines\n" // done
+ " -z, --null-data a data line ends in 0 byte, not newline\n" // yes?
+ "Miscellaneous:\n"
+ " -s, --no-messages suppress error messages\n" // done
+ " -v, --invert-match select non-matching lines\n" // done
+ " -V, --version print version information and exit\n" // done
+ " --help display this help and exit\n" // done
// +
+ " --mmap ignored for backwards compatibility\n" // no
+ "Output control:\n"
+ " -m, --max-count=NUM stop after NUM matches\n" // done - max-total
+ " -b, --byte-offset print the byte offset with output lines\n" // done
+ " -n, --line-number print line number with output lines\n" // done
+ " --line-buffered flush output on every line\n" // yes
+ " -H, --with-filename print the filename for each match\n" // done
+ " -h, --no-filename suppress the prefixing filename on output\n" // done
+ " --label=LABEL print LABEL as filename for standard input\n" // yes?
+ " -o, --only-matching show only the part of a line matching PATTERN\n" // done
+ " -q, --quiet, --silent suppress all normal output\n" // done
+ " --binary-files=TYPE assume that binary files are TYPE;\n" // yes?
+ " TYPE is `binary', `text', or `without-match'\n"
+ " -a, --text equivalent to --binary-files=text\n" // yes?
+ " -I equivalent to --binary-files=without-match\n" // yes?
+ " -d, --directories=ACTION how to handle directories;\n" // done
+ " ACTION is `read', `recurse', or `skip'\n"
+ " -D, --devices=ACTION how to handle devices, FIFOs and sockets;\n" // no
+ " ACTION is `read' or `skip'\n"
+ " -R, -r, --recursive equivalent to --directories=recurse\n" // done
+ " --include=FILE_PATTERN search only files that match FILE_PATTERN\n" // done
+ " --exclude=FILE_PATTERN skip files and directories matching FILE_PATTERN\n" // done
+ " --exclude-from=FILE skip files matching any file pattern from FILE\n" // done
+ " --exclude-dir=PATTERN directories that match PATTERN will be skipped.\n" // done
+ " -L, --files-without-match print only names of FILEs containing no match\n" // done
+ " -l, --files-with-matches print only names of FILEs containing matches\n" // done
+ " -c, --count print only a count of matching lines per FILE\n" // done
+ " -T, --initial-tab make tabs line up (if needed)\n" // no
+ " -Z, --null print 0 byte after FILE name\n" // no - use --sep=CHAR
+ "Context control:\n"
+ " -B, --before-context=NUM print NUM lines of leading context\n" // done
+ " -A, --after-context=NUM print NUM lines of trailing context\n" // done
+ " -C, --context=NUM print NUM lines of output context\n" // done
+ " -NUM same as --context=NUM\n" // no
+ " --color[=WHEN],\n" // no
+ " --colour[=WHEN] use markers to highlight the matching strings;\n" // no
+ " WHEN is `always', `never', or `auto'\n"
+ " -U, --binary do not strip CR characters at EOL (MSDOS)\n" // no?
+ " -u, --unix-byte-offsets report offsets as if CRs were not there (MSDOS)\n" // no?
+ "\n"
+ " `egrep' means `grep -E'. `fgrep' means `grep -F'.\n"
+ " Direct invocation as either `egrep' or `fgrep' is deprecated.\n"
+ " With no FILE, or when FILE is -, read standard input. If less than two FILEs\n"
+ " are given, assume -h. Exit status is 0 if any line was selected, 1 otherwise;\n"
+ " if any error occurs and -q was not given, the exit status is 2.\n"
+ "\n"
+ " Report bugs to: bug-grep@gnu.org\n"
+ " GNU Grep home page: \n"
+ " General help using GNU software: \n";
System.out.print( msg );

}

public static void version() {

System.out.println( "Java Grep 0.1" );

}

}

Friday, November 18, 2011

Java Grep

This is more of the real deal command line grep.
Not some of the crap that I've seen where they just loop through a file and execute a regex search.

I'll clean up later and make things non static and possibly implement a few other command line options.

For the most part, it matches the output of GNU Grep. I even left the GNU Grep tag in the usage() function. I'll remove that later too.

And yes code formatting is terrible.


import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;

public class grep {

private static Options grepOptions;

private static CommandLine commandLine;

private static Pattern grepPattern;

private static String regex;

static {
grepOptions = createOptions();
}


private static List filesToProcess ;

private static ArrayList< String > excludes;

public static Pattern compilePattern( final String pat ) {

int flags = 0;

String pattern = new String( pat );

if( commandLine.hasOption( 'i' ) || commandLine.hasOption( "ignore-case" ) ) {
flags |= Pattern.CASE_INSENSITIVE;
}

if( ( commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" ) )
&&
( commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" ) ) ) {
printErrorMessage( "Conflicting options (-w|--word-regexp) && (-x|--line-regexp)" );
return null;
}

if( commandLine.hasOption( 'w' ) || commandLine.hasOption( "word-regexp" ) ) {

// poor mans way to do it
pattern = "\\b" + pattern + "\\b";
}

if( commandLine.hasOption( 'x' ) || commandLine.hasOption( "line-regexp" ) ) {

// poor mans way to do it
pattern = "^" + pattern + "$";
}

return Pattern.compile( pattern, flags );

}

private static Options createOptions() {

final Options opts = new Options();

opts.addOption( "E", "extended-regexp", false, "PATTERN is an extended regular expression (ERE)" );
opts.addOption( "F", "fixed-strings", false, "PATTERN is a set of newline-separated fixed strings" );
opts.addOption( "G", "basic-regexp", false, "PATTERN is a basic regular expression (BRE)" );
opts.addOption( "P", "perl-regexp", false, "PATTERN is a Perl regular expression" );
opts.addOption( OptionBuilder.withLongOpt( "regexp" ).hasArg().withArgName( "PATTERN" )
.withDescription( "use PATTERN for matching" ).create( 'e' ) );
opts.addOption( OptionBuilder.withLongOpt( "file" ).hasArg().withArgName( "FILE" )
.withDescription( "obtain PATTERN from FILE" ).create( 'f' ) );
opts.addOption( "i", "ignore-case", false, "ignore case distinctions" );
opts.addOption( "w", "word-regexp", false, "force PATTERN to match only whole words" );
opts.addOption( "x", "line-regexp", false, "force PATTERN to match only whole lines" );
opts.addOption( "z", "null-data", false, "a data line ends in 0 byte, not newline" );
opts.addOption( "s", "no-messages", false, "suppress error messages" );
opts.addOption( "v", "invert-match", false, "select non-matching lines" );
opts.addOption( "V", "version", false, "print version information and exit" );
opts.addOption( OptionBuilder.withLongOpt( "help" ).withDescription( "display this help and exit" ).create() );
// opts.addOption( OptionBuilder.withLongOpt( "mmap" ).withDescription(
// "ignored for backwards compatibility" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "max-count" ).withArgName( "NUM" ).hasArg()
.withDescription( "stop after NUM matches" ).create( 'm' ) );
opts.addOption( "b", "byte-offset", false, "print the byte offset with output lines" );
opts.addOption( "n", "line-number", false, "print line number with output lines" );
opts.addOption( OptionBuilder.withLongOpt( "line-buffered" ).withDescription( "flush output on every line" )
.create() );
opts.addOption( "H", "with-filename", false, "print the filename for each match" );
opts.addOption( "h", "no-filename", false, "suppress the prefixing filename on output" );
opts.addOption( OptionBuilder.withLongOpt( "label" ).hasArg().withArgName( "LABEL" )
.withDescription( "print LABEL as filename for standard input" ).create() );
opts.addOption( "o", "only-matching", false, "show only the part of a line matching PATTERN" );
opts.addOption( "q", "quiet", false, "suppress all normal output" );
opts.addOption( OptionBuilder.withLongOpt( "silent" ).withDescription( "suppress all normal output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "binary-files" ).hasArg().withArgName( "TYPE" )
.withDescription( "assume that binary files are TYPE" ).create() );
// TYPE is `binary', `text', or `without-match'
opts.addOption( "a", "text", false, "equivalent to --binary-files=text" );
opts.addOption( "I", false, "equivalent to --binary-files=without-match" );
opts.addOption( OptionBuilder.withLongOpt( "directories" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle directories" ).create( 'd' ) );
// ACTION is `read', `recurse', or `skip'"
opts.addOption( OptionBuilder.withLongOpt( "devices" ).hasArg().withArgName( "ACTION" )
.withDescription( "how to handle devices, FIFOs and sockets" ).create( 'D' ) );
// ACTION is `read' or `skip'
opts.addOption( "R", false, "equivalent to --directories=recurse" );
opts.addOption( "r", "recursive", false, "equivalent to --directories=recurse" );
opts.addOption( OptionBuilder.withLongOpt( "include" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "search only files that match FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude" ).hasArg().withArgName( "FILE_PATTERN" )
.withDescription( "skip files and directories matching FILE_PATTERN" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-from" ).hasArg().withArgName( "FILE" )
.withDescription( "skip files matching any file pattern from FILE" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "exclude-dir" ).hasArg().withArgName( "PATTERN" )
.withDescription( "directories that match PATTERN will be skipped" ).create() );
opts.addOption( "L", "files-without-match", false, "print only names of FILEs containing no match" );
opts.addOption( "l", "files-with-matches", false, "print only names of FILEs containing matches" );
opts.addOption( "c", "count", false, "print only a count of matching lines per FILE" );
opts.addOption( "T", "initial-tab", false, "make tabs line up (if needed)" );
opts.addOption( "Z", "null", false, "print 0 byte after FILE name" );
opts.addOption( OptionBuilder.withLongOpt( "before-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of leading context" ).create( 'B' ) );
opts.addOption( OptionBuilder.withLongOpt( "after-context" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of trailing context" ).create( 'A' ) );
opts.addOption( OptionBuilder.withLongOpt( "contex" ).hasArg().withArgName( "NUM" )
.withDescription( "print NUM lines of output context" ).create( 'C' ) );
// * + " -NUM same as --context=NUM\n" +
opts.addOption( OptionBuilder.withLongOpt( "color" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "colour" ).hasOptionalArg().withArgName( "WHEN" )
.withDescription( "use markers to highlight the matching strings" ).create() );
// *
// " WHEN is `always', `never', or `auto'\n"
opts.addOption( "U", "binary", false, "do not strip CR characters at EOL (MSDOS)" );
opts.addOption( "u", "unix-byte-offsets", false, "report offsets as if CRs were not there (MSDOS)" );

opts.addOption( OptionBuilder.withLongOpt( "debug" ).withDescription( "turn on debugging output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "verbose" ).withDescription( "turn on verbose output" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "test" ).withDescription( "run in test mode" ).create() );
opts.addOption( OptionBuilder.withLongOpt( "logging" ).withDescription( "use nice logging, Log4J/SLF4J" )
.create() );

return opts;
}


private static void grepFileOrig( final File file ) {

long maxCount = maxCount();
long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContext = new LinkedList< String >();
List< String > afterContext = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ) ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = grepPattern.matcher( line );
boolean found = m.find();

if( ( found && !invertMatch() ) ) {
count++;
String match = line;
if( printMatchOnly() ) {
match = m.group();
}
printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext );
if( printFileNameOnly() ) {
return;
}
//
}
else if( !found && invertMatch() ) {
count++;
printMatch( file, line, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext );
// TODO: this has a slightly different meaning
if( printFileNameOnly() ) {
return;
}
}

if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContext.add( line );
if( beforeContext.size() > beforeContext() ) {
beforeContext.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch() ) {
printMessage( file.getName() );
}
}

if( printCountOnly() ) {
printMessage( ( file.getName() + ":" + count ) );
}

}

private static void printMessage( final String msg ) {

if( !quiet() ) {
System.out.println( msg );
}
}

private static void grepFiles() {

// at this point the list was expanded into file names
for( final File f : filesToProcess ) {

grepFile( f );
}

}

public static boolean invertMatch() {
return commandLine.hasOption( 'v' ) || commandLine.hasOption( "invert-match" );
}

public static void main( final String[] args ) {

reset();

if( ArrayUtils.isEmpty( args ) ) {
usage();
return;
}

// parse the command line
try {
final CommandLineParser parser = new PosixParser();
commandLine = parser.parse( grepOptions, args );
}
catch( final ParseException pe ) {
System.out.print( pe.getMessage() );
usage();
return;
}

// if version or help was given then print and quit
if( commandLine.hasOption( "help" ) ) {
usage();
return;
}

if( commandLine.hasOption( "version" ) || commandLine.hasOption( 'V' ) ) {
version();
return;
}

//
if( commandLine.hasOption( 'e' ) || commandLine.hasOption( "regexp" ) ) {
String r = commandLine.getOptionValue( 'e' );
if( r != null ) {
regex = r;
}
else {
usage();
return;
}
}

if( commandLine.hasOption( 'f' ) || commandLine.hasOption( "file" ) ) {
if( StringUtils.isNotEmpty( regex ) ) {
// cannot specify both
usage();
return;
}

String r = readRegexFromFile();
if( StringUtils.isEmpty( r ) ) {
usage();
return;
}
regex = r;
}

if( commandLine.hasOption( "exclude-from" ) ) {
readExcludeFrom();
if( CollectionUtils.isEmpty( excludes ) ) {
usage();
return;
}
}

// get the unprocessed items
final List argList = commandLine.getArgList();

// we must have some args besides options
if( CollectionUtils.isEmpty( argList ) ) {
usage();
return;
}

// just to get things started
// we are assuming: [OPTION]... PATTERN [FILE]...
// options are pulled out by the command line processor
int fileOffset = 0;
if( StringUtils.isEmpty( regex ) ) {
regex = (String)argList.get( 0 );
fileOffset++;
}

// make sure its a good regex
try {
grepPattern = compilePattern( regex );
}
catch( final PatternSyntaxException pse ) {
printErrorMessage( "invalid regex syntax\nregex:" + regex + "\n" + pse.getMessage() );
return;
}

if( grepPattern == null ) {
return;
}

// just to get things going
processFileArgs( argList.subList( fileOffset, argList.size() ) );

// ok lets start
grepFiles();

}

private static void readExcludeFrom() {

String val = commandLine.getOptionValue( "exclude-from" );
if( StringUtils.isEmpty( val ) ) {
// trouble
return;
}

try( BufferedReader br = new BufferedReader( new FileReader( new File( val ) ) ) ) {
excludes = new ArrayList< String >();

for( String line = br.readLine(); line != null; line = br.readLine() ) {
excludes.add( line );
}

}
catch( IOException e ) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

private static long maxCount() {
if( commandLine.hasOption( 'm' ) || commandLine.hasOption( "max-count" ) ){
String s = commandLine.getOptionValue( 'm' );
return Long.parseLong( s );
}

return 0;
}

private static boolean printByteOffset() {

return commandLine.hasOption( 'b' ) || commandLine.hasOption( "byte-offset" );

}

public static void printErrorMessage( final String msg ) {
if( !( commandLine.hasOption( 's' ) || commandLine.hasOption( "no-messages" ) ) ) {
System.out.println(msg);
}
}

private static boolean printFileName() {

boolean foundH = commandLine.hasOption( 'H' ) || commandLine.hasOption( "with-filename" );
if( foundH ) {
return true;
}
boolean foundh = commandLine.hasOption( 'h' ) || commandLine.hasOption( "no-filename" );
if( foundh ) {
return false;
}

// what other factors?
return true;
}

private static boolean printFileNameOnly() {

return( commandLine.hasOption( 'l' ) || commandLine.hasOption( "files-with-matches" ) );
}

private static boolean printLineNumber() {

return( commandLine.hasOption( 'n' ) || commandLine.hasOption( "line-number" ) );
}

private static void printMatch( final File file, final String line, final long lineNumber,
final long count, final long byteOffset, final List< String > beforeContext, final List< String > afterContext ) {

if( quiet() || printFilesWithoutMatch() || printCountOnly() ) {
return;
}

/*
* if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
* System.out.println( "--" );
* }
*/

if( beforeContext() > 0 ) {
for( String bc : beforeContext ) {
String s = new String( bc );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

String msg = "";
if( printFileNamesOnly() ) {
try {
System.out.println( String.format( "%1$s", file.getCanonicalPath() ) );
}
catch( IOException ioe ) {
// TODO:
}
return;
}

if( printFileName() && ( filesToProcess.size() > 1 ) ) {
try {
msg += file.getCanonicalPath() + ":";
}
catch( IOException ioe ) {}
}

if( printLineNumber() ) {
msg += lineNumber + ":";
}

if( printByteOffset() ) {
msg += byteOffset + ":";
}

msg = msg.length() > 1 ? msg + " " + line : line;

System.out.println( msg );

if( afterContext() > 0 ) {
for( String ac : afterContext ) {
String s = new String( ac );
if( filesToProcess.size() > 1 ) {
try {
s = file.getCanonicalPath() + "-" + s;
}
catch( IOException ioe ) {}
}
System.out.println( s );
}
}

if( ( beforeContext() > 0 ) || ( afterContext() > 0 ) ) {
System.out.println( "--" );
}
}

private static boolean printMatchOnly() {

return commandLine.hasOption( 'o' ) || commandLine.hasOption( "only-matching" );

}

// TODO: note that file names can contain wild cards too
// so this method should also expand directories as well
private static void processFileArgs( final List list ) {

filesToProcess = new ArrayList< File >();

for( final Object o : list ) {
final String name = (String)o;

final File f = new File( name );

if( f.exists() ) {
if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
filesToProcess.add( f );
}
}
}
else if( f.isDirectory() ) {
if( recurseDirectories() ) {
filesToProcess.addAll( recurseDir( f ) );
}
}
}
}

}

private static List< File > recurseDir( final File dir ) {

List< File > files = new ArrayList< File >();

for( File f : dir.listFiles() ) {

if( f.isFile() ) {
if( includeFile( f ) ) {
if( !excludeFile( f ) ) {
files.add( f );
}
}
}
else if( f.isDirectory() ) {
if( !excludeDir( f ) ) {
files.addAll( recurseDir( f ) );
}
}

}

return files;

}

private static boolean recurseDirectories() {

if( commandLine.hasOption( 'd' ) || commandLine.hasOption( "directories" ) ) {

String val = commandLine.getOptionValue( 'd' );
if( StringUtils.endsWithIgnoreCase( val, "recurse" ) ) {
return true;
}

}

if( commandLine.hasOption( 'R' )
|| commandLine.hasOption( 'r' )
|| commandLine.hasOption( "recursive" ) ) {
return true;
}

return false;
}

private static boolean quiet() {

return commandLine.hasOption( 'q' ) || commandLine.hasOption( "--quiet" ) || commandLine.hasOption( "silent" );

}

private static String readRegexFromFile() {
String val = null;

String f = commandLine.getOptionValue( 'f' );
File file = new File(f);
if( file.exists() && file.isFile() ) {

try( BufferedReader br = new BufferedReader( new FileReader( f ) ) ){

String line = br.readLine();
if( StringUtils.isNotBlank( line ) ) {
val = line ;
}

}
catch( IOException e ) {
printErrorMessage( "Cannot open file:" + f );
}
}

return val ;
}

private static void reset() {

commandLine = null;
grepPattern = null;
regex = null;
filesToProcess = null;
excludes = null;


}

public static void usage() {

final String msg =
"Usage: grep [OPTION]... PATTERN [FILE]...\n" + "Search for PATTERN in each FILE or standard input.\n"
+ "PATTERN is, by default, a basic regular expression (BRE).\n"
+ "Example: grep -i 'hello world' menu.h main.c\n"
+ "Regexp selection and interpretation:\n"
+ " -E, --extended-regexp PATTERN is an extended regular expression (ERE)\n" // no
+ " -F, --fixed-strings PATTERN is a set of newline-separated fixed strings\n" // no
+ " -G, --basic-regexp PATTERN is a basic regular expression (BRE)\n" // no
+ " -P, --perl-regexp PATTERN is a Perl regular expression\n" // no
+ " -e, --regexp=PATTERN use PATTERN for matching\n" // done
+ " -f, --file=FILE obtain PATTERN from FILE\n" // done
+ " -i, --ignore-case ignore case distinctions\n" // done
+ " -w, --word-regexp force PATTERN to match only whole words\n" // done
+ " -x, --line-regexp force PATTERN to match only whole lines\n" // done
+ " -z, --null-data a data line ends in 0 byte, not newline\n" // yes?
+ "Miscellaneous:\n"
+ " -s, --no-messages suppress error messages\n" // done
+ " -v, --invert-match select non-matching lines\n" // done
+ " -V, --version print version information and exit\n" // done
+ " --help display this help and exit\n" // done
// +
+ " --mmap ignored for backwards compatibility\n" // no
+ "Output control:\n"
+ " -m, --max-count=NUM stop after NUM matches\n" // done - max-total
+ " -b, --byte-offset print the byte offset with output lines\n" // done
+ " -n, --line-number print line number with output lines\n" // done
+ " --line-buffered flush output on every line\n" // yes
+ " -H, --with-filename print the filename for each match\n" // done
+ " -h, --no-filename suppress the prefixing filename on output\n" // done
+ " --label=LABEL print LABEL as filename for standard input\n" // yes?
+ " -o, --only-matching show only the part of a line matching PATTERN\n" // done
+ " -q, --quiet, --silent suppress all normal output\n" // done
+ " --binary-files=TYPE assume that binary files are TYPE;\n" // yes?
+ " TYPE is `binary', `text', or `without-match'\n"
+ " -a, --text equivalent to --binary-files=text\n" // yes?
+ " -I equivalent to --binary-files=without-match\n" // yes?
+ " -d, --directories=ACTION how to handle directories;\n" // done
+ " ACTION is `read', `recurse', or `skip'\n"
+ " -D, --devices=ACTION how to handle devices, FIFOs and sockets;\n" // no
+ " ACTION is `read' or `skip'\n"
+ " -R, -r, --recursive equivalent to --directories=recurse\n" // done
+ " --include=FILE_PATTERN search only files that match FILE_PATTERN\n" // done
+ " --exclude=FILE_PATTERN skip files and directories matching FILE_PATTERN\n" // done
+ " --exclude-from=FILE skip files matching any file pattern from FILE\n" // done
+ " --exclude-dir=PATTERN directories that match PATTERN will be skipped.\n" // done
+ " -L, --files-without-match print only names of FILEs containing no match\n" // done
+ " -l, --files-with-matches print only names of FILEs containing matches\n" // done
+ " -c, --count print only a count of matching lines per FILE\n" // done
+ " -T, --initial-tab make tabs line up (if needed)\n" // no
+ " -Z, --null print 0 byte after FILE name\n" // no - use --sep=CHAR
+ "Context control:\n"
+ " -B, --before-context=NUM print NUM lines of leading context\n" // done
+ " -A, --after-context=NUM print NUM lines of trailing context\n" // done
+ " -C, --context=NUM print NUM lines of output context\n" // done
+ " -NUM same as --context=NUM\n" // no
+ " --color[=WHEN],\n" // no
+ " --colour[=WHEN] use markers to highlight the matching strings;\n" // no
+ " WHEN is `always', `never', or `auto'\n"
+ " -U, --binary do not strip CR characters at EOL (MSDOS)\n" // no?
+ " -u, --unix-byte-offsets report offsets as if CRs were not there (MSDOS)\n" // no?
+ "\n"
+ " `egrep' means `grep -E'. `fgrep' means `grep -F'.\n"
+ " Direct invocation as either `egrep' or `fgrep' is deprecated.\n"
+ " With no FILE, or when FILE is -, read standard input. If less than two FILEs\n"
+ " are given, assume -h. Exit status is 0 if any line was selected, 1 otherwise;\n"
+ " if any error occurs and -q was not given, the exit status is 2.\n"
+ "\n"
+ " Report bugs to: bug-grep@gnu.org\n"
+ " GNU Grep home page: \n"
+ " General help using GNU software: \n";
System.out.print( msg );

}

private static void version() {

System.out.println("Java Grep 0.1");

}

public static boolean includeFile( final File f ) {

if( commandLine.hasOption( "include" ) ) {

String pat = commandLine.getOptionValue( "include" );
String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, pat );

}

return true;
}

public static boolean excludeFile( final File f ) {

if( commandLine.hasOption( "exclude" ) ) {
String pat = commandLine.getOptionValue( "exclude" );
String nm = FilenameUtils.getName( f.getName() );
return FilenameUtils.wildcardMatch( nm, pat );
}

if( commandLine.hasOption( "exclude-from" ) ) {
for( String pat : excludes ) {
String nm = FilenameUtils.getName( f.getName() );
boolean ex = FilenameUtils.wildcardMatch( nm, pat );
if( ex ) {
return true;
}
}
}

return false ;

}

private static boolean excludeDir( final File f ) {

if( commandLine.hasOption( "exclude-dir" ) ) {
String pat = commandLine.getOptionValue( "exclude-dir" );
if( FilenameUtils.wildcardMatch( FilenameUtils.getName( f.getName() ), pat ) ) {
return true;
}
}

return false;
}

private static boolean printFileNamesOnly() {

return printFilesWithoutMatch() || printFileNameOnly();
}

private static boolean printFilesWithoutMatch() {

return commandLine.hasOption( 'L' ) || commandLine.hasOption( "files-without-match" );
}

private static boolean printCountOnly() {

return commandLine.hasOption( 'c' ) || commandLine.hasOption( "count" );
}

private static long beforeContext() {

if( commandLine.hasOption( 'B' ) || commandLine.hasOption( "before-context" ) ) {
String s = commandLine.getOptionValue( 'B' );
return Long.parseLong( s );
}

return context();
}

private static long afterContext() {

if( commandLine.hasOption( 'A' ) || commandLine.hasOption( "after-context" ) ) {
String s = commandLine.getOptionValue( 'A' );
return Long.parseLong( s );
}

return context();
}

private static long context() {

if( commandLine.hasOption( 'C' ) || commandLine.hasOption( "context" ) ) {
String s = commandLine.getOptionValue( 'C' );
return Long.parseLong( s );
}

return 0;

}

private static void grepFile( final File file ) {

long maxCount = maxCount();
long lineNumber = 0;
long count = 0;
long byteOffset = 0;
List< String > beforeContext = new LinkedList< String >();
List< String > afterContext = new LinkedList< String >();

// keep it simple for now
try( BufferedReader bfr = new BufferedReader( new FileReader( file ), 16000 ) ) {

for( String line = bfr.readLine(); line != null; line = bfr.readLine() ) {

lineNumber++;
Matcher m = grepPattern.matcher( line );
boolean found = m.find();

if( ( found && !invertMatch() ) ) {
count++;
if( afterContext() > 0 ) {
afterContext.clear();
bfr.mark( 4000 );
for( int i = 0; i < afterContext(); i++ ) { String s = bfr.readLine(); if( s != null ) { afterContext.add( s ); } else { break; } } bfr.reset(); } String match = line; if( printMatchOnly() ) { match = m.group(); } printMatch( file, match, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext ); if( printFileNameOnly() ) { return; } // } else if( !found && invertMatch() ) { count++; printMatch( file, line, lineNumber, count, ( byteOffset + m.start() ), beforeContext, afterContext ); // TODO: this has a slightly different meaning if( printFileNameOnly() ) { return; } } if( ( maxCount != 0 ) && ( count >= maxCount ) ) {
break;
}

byteOffset += line.getBytes().length;
byteOffset += 1; // TODO: end of line char - what about DOS/Win32?

beforeContext.add( line );
if( beforeContext.size() > beforeContext() ) {
beforeContext.remove( 0 );
}
}
}
catch( IOException ioe ) {
System.out.println( ioe.getMessage() );
}

if( count == 0 ) {
// no matches in file
if( printFilesWithoutMatch() ) {
printMessage( file.getName() );
}
}

if( printCountOnly() ) {
printMessage( ( file.getName() + ":" + count ) );
}

}
}

Friday, July 8, 2011

Scala Collections Rant

Scala already has knocks against it for being confusing. So, I'll just kick it while it's down.

My latest frustration is with the mess that it calls Collections.

All I wanted to do was remove an item from a list. A very simple operation in just about every other programming environment ever created. I understand and appreciate that removing an item from an immutable collection should return a new list. In fact, that's exactly what I wanted to do.

So, I dug a little deeper in the Scala doc for List. But wait which List? Immutable List, which is just List. Mutable? Parrallel List? DoublyLinkedList? ListBuffer? Ok, this isn't so bad, I understand there are needs for all these.

So, lets just stick with Immutable List. Which is just called List. Even though Mutable List is called MutableList. Lets over look that too. For now.

OK. Open the Scala Doc for ImmutableList, I mean just List.

Ah, there's my remove method. But wait it's deprecated. Hmm. OK, maybe I'll use a MutableList. But wait, there's no remove method. What, no remove method on a MutableList? That's strange. Maybe its named something else? Nope. Maybe its an operator? Nope. I can contains, get, and indexOf on a MutableList. I can + an element onto a MutableList. But I can't - an element? I can also add items with + :+ +: +=: += . But not one operator to remove? What's up with that? Are these guys high on monads?

Ah, then there is a handy suggestion to use the filter function to remove the item I don't want. Good idea, except maybe I want to remove a specific item at a specific spot. Sounds bad doesn't it.

But all this functionality IS available. I was just looking at the wrong List. Its in ListBuffer. Whew. I guess I'll use a ListBuffer then. But now, I've just lost my the ImmutableLikeness of my List.

Or maybe I could have used an Immutable ListSet, just ListSet. But there is a possibility that I might want duplicates in the List. So I want a remove(index) function just like the measly old Java Util List. Which by the way must be jealous that it only has 20 methods vs 250 methods for the Scala List on steroids.

I could roll my own function. Or use some combination of take, head tail, etc. Or filter. But come on, its just remove.

So, that lead me to think about the API. List and MutableList I thought would basically have the same interface. Right? ... Well, right?

This discussion could get really long now.

A List is a LinearSeqOptimized, Product, LinearSeq, LinearSeq, LinearSeqLike, Seq, Seq, SeqLike, GenSeq, GenSeqLike, PartialFunction, Function1, Iterable, Iterable, IterableLike, Equals, GenIterable, GenIterableLike, Traversable, Immutable, Traversable, GenTraversable, GenericTraversableTemplate, TraversableLike, GenTraversableLike, Parallelizable, TraversableOnce, GenTraversableOnce, FilterMonadic, HasNewBuilder, AnyRef, Any.

And a MutableList is a Serializable, Serializable, Builder, Growable,
LinearSeqOptimized, LinearSeq, LinearSeq, LinearSeqLike, Seq, SeqLike, Cloneable, Seq, SeqLike, GenSeq, GenSeqLike, PartialFunction, Function1, Iterable, Iterable,
IterableLike, Equals, GenIterable, GenIterableLike, Traversable, Mutable, Traversable, GenTraversable, GenericTraversableTemplate, TraversableLike, GenTraversableLike, Parallelizable, TraversableOnce, GenTraversableOnce, FilterMonadic, HasNewBuilder, AnyRef, Any

That must explain the difference. Wait what.

Among 31 other things, a List is also a Function1? And a List is a HasNewBuilder? Isn't HasNewBuilder a question? Shouldn't that be NewBuilderLike? Maybe Buildable?

A MutableList is a Growable. But not a Shrinkable? Heresy.

And when did -like replace -able? Is this the new Valley Girl Moon Unit Zappa naming convention?

And it opens the possibility for the Likeable trait.

I really enjoy the name TraversableLike.

Why not use -ish? Then we can have TraverableLikeIsh.

Maybe throw in an -ly occasionally too?

So a List is TraverableOnce. Doesn't that mean the second time I try to loop over it. My code will fail?

Sausage, Politicians.

Now lets add Scala Collections.