Merge pull request WorldBank-Transport#260 from flibbertigibbet/featu…

…re/move-jvm-opts

Feature/move jvm opts

deployment/provision.sh

@@ -241,13 +241,13 @@ else
     popd
     echo 'Tuning Postgres'
     # set SHMMAX for postgres shared buffers
-    cat "kernel.shmmax = 107374182400" >> /etc/sysctl.conf
+    echo "kernel.shmmax = 107374182400" >> /etc/sysctl.conf
     sysctl -p
     pushd /etc/postgresql/9.1/main
         sed -i '/shared_buffers =/c\shared_buffers = 756MB' postgresql.conf
         sed -i '/max_connections =/c\max_connections = 30' postgresql.conf
         sed -i '/effective_cache_size =/c\effective_cache_size = 2GB' postgresql.conf
-        sed -i '/work_mem =/c\work_mem = 64MB' postgresql.conf
+        sed -i '/#work_mem =/c\work_mem = 64MB' postgresql.conf
         sed -i '/maintenance_work_mem =/c\maintenance_work_mem = 512MB' postgresql.conf
         sed -i '/temp_buffers =/c\temp_buffers = 32MB' postgresql.conf
         service postgresql restart
@@ -580,7 +580,7 @@ kill timeout 30
 script
     echo \$\$ > /var/run/oti-indicators.pid
     chdir $SCALA_ROOT
-    exec ./sbt 'project opentransit' -mem $SBT_MEM_MB -XX:-UseConcMarkSweepGC -XX:+UseGCOverheadLimit run
+    exec ./sbt 'project opentransit' -mem $SBT_MEM_MB run
 end script
 
 pre-stop script

scala/.jvmopts

@@ -0,0 +1,5 @@
+-Dfile.encoding=UTF8
+-XX:MaxPermSize=4g
+-XX:-UseConcMarkSweepGC
+-XX:+UseGCOverheadLimit
+-XX:+CMSClassUnloadingEnabled

scala/gtfs/src/main/scala/com/azavea/gtfs/Timer.scala

@@ -0,0 +1,13 @@
+package com.azavea.gtfs
+
+object Timer {
+  // Time how long a block takes to execute.  From here:
+  // http://stackoverflow.com/questions/9160001/how-to-profile-methods-in-scala
+  def timedTask[R](msg: String)(block: => R): R = {
+    val t0 = System.currentTimeMillis
+    val result = block    // call-by-name
+    val t1 = System.currentTimeMillis
+    println(msg + " in " + ((t1 - t0) / 1000.0) + " s")
+    result
+  }
+}

scala/gtfs/src/main/scala/com/azavea/gtfs/io/csv/AgencyFile.scala

@@ -10,7 +10,6 @@ object AgencyFile extends GtfsFile[Agency] {
 
   def parse(path: String): Seq[Agency] = {
     val agencies = mutable.ListBuffer[Agency]()
-
     for (s <- CsvParser.fromPath(path)) {
       agencies +=
       Agency(

scala/gtfs/src/main/scala/com/azavea/gtfs/io/csv/FrequenciesFile.scala

@@ -9,6 +9,7 @@ object FrequenciesFile extends GtfsFile[FrequencyRecord] {
 
   def parse(path: String): Seq[FrequencyRecord] =
     (for (f <- CsvParser.fromPath(path)) yield {
+      println("parsing a FrequencyRecord")
       FrequencyRecord(
         f("trip_id").get.intern,
         f("start_time").get,

scala/gtfs/src/main/scala/com/azavea/gtfs/io/database/DatabaseRecordImport.scala

@@ -1,6 +1,7 @@
 package com.azavea.gtfs.io.database
 
 import com.azavea.gtfs._
+import com.azavea.gtfs.Timer.timedTask
 
 import scala.slick.jdbc.JdbcBackend.Session
 
@@ -32,21 +33,25 @@ class DatabaseRecordImport(val geomColumnName: String = Profile.defaultGeomColum
 
   def load(records: GtfsRecords, clobber: Boolean = true): Unit = {
     if(clobber) deleteAll
-
+    
     def ensureNoNullPeriods(stopTime: StopTimeRecord) = {
       assert(stopTime.arrivalTime != null)
       assert(stopTime.departureTime != null)
       stopTime
     }
 
-    load(records.agencies, agenciesTable)
-    load(records.stops, stopsTable)
-    load(records.calendarDateRecords, calendarDateRecordsTable)
-    load(records.calendarRecords, calendarRecordsTable)
-    load(records.routeRecords, routeRecordsTable)
-    load(records.tripRecords, tripRecordsTable)
-    load(records.stopTimeRecords.view.map(ensureNoNullPeriods), stopTimeRecordsTable)
-    load(records.frequencyRecords, frequencyRecordsTable)
-    load(records.tripShapes, tripShapesTable)
+    timedTask("loaded agencies") { load(records.agencies, agenciesTable) }
+    timedTask("loaded stops") { load(records.stops, stopsTable) }
+    timedTask("loaded calendar dates") {
+      load(records.calendarDateRecords, calendarDateRecordsTable)
+    }
+    timedTask("loaded calendar") { load(records.calendarRecords, calendarRecordsTable) }
+    timedTask("loaded routes") { load(records.routeRecords, routeRecordsTable) }
+    timedTask("loaded trips") { load(records.tripRecords, tripRecordsTable) }
+    timedTask("loaded stop times") {
+      load(records.stopTimeRecords.view.map(ensureNoNullPeriods), stopTimeRecordsTable)
+    }
+    timedTask("loaded frequencies") { load(records.frequencyRecords, frequencyRecordsTable) }
+    timedTask("loaded trip shapes") { load(records.tripShapes, tripShapesTable) }
   }
 }

scala/gtfs/src/main/scala/com/azavea/gtfs/io/database/StopTimeRecordsTable.scala

@@ -7,6 +7,7 @@ import geotrellis.slick._
 
 trait StopTimeRecordsTable { this: Profile  =>
   import profile.simple._
+
 
   class StopTimeRecords(tag: Tag) extends Table[StopTimeRecord](tag, "gtfs_stop_times") {
     def stop_id = column[String]("stop_id")
@@ -19,5 +20,6 @@ trait StopTimeRecordsTable { this: Profile  =>
     def * = (stop_id, trip_id, stop_sequence, arrival_time, departure_time, shape_dist_traveled) <>
       (StopTimeRecord.tupled, StopTimeRecord.unapply)
   }
+
   val stopTimeRecordsTable = TableQuery[StopTimeRecords]
 }

scala/opentransit/src/main/scala/com/azavea/opentransit/io/GtfsIngest.scala

@@ -54,7 +54,9 @@ object GtfsIngest {
     geomTransform(srid, "utm_datasources_demographicdatafeature", "MultiPolygon", "geom")
 
     println("Finished transforming to local UTM zone.")
-    records.routeRecords.size
+    val routeSize = records.routeRecords.size
+    println("All done processing feed in GtfsIngest!")
+    routeSize
   }
 
 

scala/opentransit/src/main/scala/com/azavea/opentransit/service/IngestRoute.scala

@@ -1,5 +1,7 @@
 package com.azavea.opentransit.service
 
+import com.azavea.gtfs.Timer.timedTask
+
 import com.azavea.opentransit._
 import com.azavea.opentransit.io.GtfsIngest
 
@@ -24,7 +26,7 @@ trait IngestRoute extends Route { self: DatabaseInstance =>
               println(s"parsing GTFS data from: $gtfsDir")
               val routeCount =
                 db withSession { implicit session =>
-                  GtfsIngest(gtfsDir)
+                  timedTask("Ingested GTFS") { GtfsIngest(gtfsDir) }
                 }
 
               JsObject(