Monitoring JVM

Advertisements

Scala REPL in Notebook

Scalaz in the REPL

First run sbt:

sbt

Then issue these commands:

set scalaVersion := “2.10.0”
set libraryDependencies += “org.scalaz” %% “scalaz-core” % “7.0.0”
set initialCommands += “import scalaz._, Scalaz._”
session save
console

Algebird Monoids for IP Addresses and counts in Scala

import com.twitter.algebird.Operators._


case class IPRecord(val ipAddress: String, val number: Int) extends Ordered[IPRecord] {
 def compare(that: IPRecord): Int = {
   val c = this.number - that.number
   if (c == 0) this.ipAddress.compareTo(that.ipAddress) else c
 }
}


val oneOneOneOne = IPRecord("1.1.1.1", 67391)
val twoTwoTwoTwo = IPRecord("2.2.2.2", 48013573)
val threeThreeThreeThree = IPRecord("3.3.3.3", 6470)
val fourFourFourFour = IPRecord("4.4.4.4", 731)

val topIPAddress: Max[IPRecord] = Max(oneOneOneOne) + Max(twoTwoTwoTwo) + Max(threeThreeThreeThree) + Max(fourFourFourFour)
assert(topIPAddress.get == twoTwoTwoTwo)

Simple Apache Auth Log Processing with Spark job

/* 

SimpleApp.scala

Simple Spark Job for processing Apache auth.log for Invalid user login attempts and Failed password counts
./bin/spark-submit --class "SimpleApp" --master local[4] target/scala-2.10/simple-project_2.10-1.0.jar

 */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {
 def main(args: Array[String]) {
 val logFile = "/Users/antigen/Downloads/sanitized_log/auth.log" 
 val conf = new SparkConf().setAppName("SimpleApacheLogProcessing Application")
 val sc = new SparkContext(conf)
 val logData = sc.textFile(logFile, 2).cache()
 val numAs = logData.filter(line => line.contains("Invalid user")).count()
 val numBs = logData.filter(line => line.contains("Failed password")).count()
 println("Lines with INVALID USER: %s, Lines with FAILED PASSWORD: %s".format(numAs, numBs))
 }
}


Code, Folder Structure, simple.sbt, and packaged jar files here:
https://github.com/bigsnarfdude/SimpleApp
http://databricks.gitbooks.io/databricks-spark-reference-applications/content/logs_analyzer/app/README.html

Data Science Stack

Finally got Algebird and Apache Log Parsing libraries into my Apache Spark REPL

 

https://github.com/twitter/algebird/wiki/Learning-Algebird-Monoids-with-REPL