From 199e1cfac183f896e003a8af753c6fd21b7b100a Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 23 Aug 2016 19:30:47 +0800 Subject: [PATCH 001/116] lesson4 complete --- build.sbt | 2 ++ src/main/scala/lesson2And3And4/MainApp.scala | 7 ++++++- src/main/scala/lesson2And3And4/SourceCode.scala | 2 +- src/test/scala/lesson2And3And4/SourceCodeSpec.scala | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 1585ece..2c81d60 100644 --- a/build.sbt +++ b/build.sbt @@ -88,6 +88,8 @@ exportJars := true // only show stack traces up to the first sbt stack frame traceLevel := 0 +mainClass in assembly := Some("lesson2And3And4.MainApp") + // add SWT to the unmanaged classpath // unmanagedJars in Compile += file("/usr/share/java/swt.jar") diff --git a/src/main/scala/lesson2And3And4/MainApp.scala b/src/main/scala/lesson2And3And4/MainApp.scala index 4a17864..4aad3a1 100644 --- a/src/main/scala/lesson2And3And4/MainApp.scala +++ b/src/main/scala/lesson2And3And4/MainApp.scala @@ -1,5 +1,10 @@ package lesson2And3And4 object MainApp extends App{ - println("welcome to use my code analyzer") + if(args.length < 1){ + println("usage: CodeAnalyzer FilePath") + }else{ + val sourceCode = SourceCode.fromFile(args(0)) + println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") + } } diff --git a/src/main/scala/lesson2And3And4/SourceCode.scala b/src/main/scala/lesson2And3And4/SourceCode.scala index ada9584..a299f19 100644 --- a/src/main/scala/lesson2And3And4/SourceCode.scala +++ b/src/main/scala/lesson2And3And4/SourceCode.scala @@ -10,7 +10,7 @@ object SourceCode{ def fromFile(path: Path):SourceCode = { import scala.io._ - val source = Source.fromFile("/Users/twer/source/scala/CodeAnalyzerTutorial/build.sbt") + val source = Source.fromFile(path) val lines = source.getLines.toList new SourceCode(path,path.split("/").last, lines) } diff --git a/src/test/scala/lesson2And3And4/SourceCodeSpec.scala b/src/test/scala/lesson2And3And4/SourceCodeSpec.scala index f0e3d94..63dbbdd 100644 --- a/src/test/scala/lesson2And3And4/SourceCodeSpec.scala +++ b/src/test/scala/lesson2And3And4/SourceCodeSpec.scala @@ -5,9 +5,9 @@ import org.scalatest.{FunSpec, ShouldMatchers} class SourceCodeSpec extends FunSpec with ShouldMatchers{ describe("SourceCode object"){ it("can read file and create a SourceCode instance"){ - val sourceCode = SourceCode.fromFile("test/resources/sourceFileSample") + val sourceCode = SourceCode.fromFile("./src/test/resources/sourceFileSample") sourceCode.name shouldBe "sourceFileSample" - sourceCode.path shouldBe "test/resources/sourceFileSample" + sourceCode.path shouldBe "./src/test/resources/sourceFileSample" sourceCode.count shouldBe 108 } } From 4c08445c3eec066c7b84103ba78ee100c8b534cb Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 28 Aug 2016 09:13:57 +0800 Subject: [PATCH 002/116] code format --- src/main/scala/lesson2And3And4/SourceCode.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/scala/lesson2And3And4/SourceCode.scala b/src/main/scala/lesson2And3And4/SourceCode.scala index a299f19..dea4281 100644 --- a/src/main/scala/lesson2And3And4/SourceCode.scala +++ b/src/main/scala/lesson2And3And4/SourceCode.scala @@ -1,17 +1,17 @@ package lesson2And3And4 -class SourceCode(val path: String, val name: String, private val lines: List[String]){ - def count:Int = lines.length +class SourceCode(val path: String, val name: String, private val lines: List[String]) { + def count: Int = lines.length } -object SourceCode{ +object SourceCode { type Path = String - def fromFile(path: Path):SourceCode = { + def fromFile(path: Path): SourceCode = { import scala.io._ val source = Source.fromFile(path) val lines = source.getLines.toList - new SourceCode(path,path.split("/").last, lines) + new SourceCode(path, path.split("/").last, lines) } } \ No newline at end of file From 9e10ccadba47128f09ed91a5dc0a445b95d0fca4 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 28 Aug 2016 16:29:02 +0800 Subject: [PATCH 003/116] change package name --- src/main/scala/{lesson2And3And4 => tutor}/MainApp.scala | 2 +- src/main/scala/{lesson2And3And4 => tutor}/SourceCode.scala | 2 +- src/test/scala/{lesson2And3And4 => tutor}/SourceCodeSpec.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/main/scala/{lesson2And3And4 => tutor}/MainApp.scala (90%) rename src/main/scala/{lesson2And3And4 => tutor}/SourceCode.scala (93%) rename src/test/scala/{lesson2And3And4 => tutor}/SourceCodeSpec.scala (94%) diff --git a/src/main/scala/lesson2And3And4/MainApp.scala b/src/main/scala/tutor/MainApp.scala similarity index 90% rename from src/main/scala/lesson2And3And4/MainApp.scala rename to src/main/scala/tutor/MainApp.scala index 4aad3a1..0883083 100644 --- a/src/main/scala/lesson2And3And4/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,4 +1,4 @@ -package lesson2And3And4 +package tutor object MainApp extends App{ if(args.length < 1){ diff --git a/src/main/scala/lesson2And3And4/SourceCode.scala b/src/main/scala/tutor/SourceCode.scala similarity index 93% rename from src/main/scala/lesson2And3And4/SourceCode.scala rename to src/main/scala/tutor/SourceCode.scala index dea4281..8e99378 100644 --- a/src/main/scala/lesson2And3And4/SourceCode.scala +++ b/src/main/scala/tutor/SourceCode.scala @@ -1,4 +1,4 @@ -package lesson2And3And4 +package tutor class SourceCode(val path: String, val name: String, private val lines: List[String]) { def count: Int = lines.length diff --git a/src/test/scala/lesson2And3And4/SourceCodeSpec.scala b/src/test/scala/tutor/SourceCodeSpec.scala similarity index 94% rename from src/test/scala/lesson2And3And4/SourceCodeSpec.scala rename to src/test/scala/tutor/SourceCodeSpec.scala index 63dbbdd..e3c56aa 100644 --- a/src/test/scala/lesson2And3And4/SourceCodeSpec.scala +++ b/src/test/scala/tutor/SourceCodeSpec.scala @@ -1,4 +1,4 @@ -package lesson2And3And4 +package tutor import org.scalatest.{FunSpec, ShouldMatchers} From 72aa4f1bb2b7e9c5670312512bba3e1af29e20dd Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 28 Aug 2016 18:04:57 +0800 Subject: [PATCH 004/116] change main class according to package renaming --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 2c81d60..7d7eaf1 100644 --- a/build.sbt +++ b/build.sbt @@ -88,7 +88,7 @@ exportJars := true // only show stack traces up to the first sbt stack frame traceLevel := 0 -mainClass in assembly := Some("lesson2And3And4.MainApp") +mainClass in assembly := Some("tutor.MainApp") // add SWT to the unmanaged classpath // unmanagedJars in Compile += file("/usr/share/java/swt.jar") From e3484339680d9a848397681ee8f2907e45133750 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 28 Aug 2016 18:05:25 +0800 Subject: [PATCH 005/116] complete directory scanner --- src/main/scala/tutor/DirectoryScanner.scala | 38 +++++++++++++++++++ src/main/scala/tutor/MainApp.scala | 17 ++++++++- src/main/scala/tutor/SourceCode.scala | 8 ++-- src/main/scala/tutor/package.scala | 2 + src/main/scala/tutor/utils/FileUtil.scala | 17 +++++++++ src/test/resources/sub/SomeCode.scala | 16 ++++++++ src/test/resources/sub/sub1/othercode.scala | 16 ++++++++ .../scala/tutor/DirectoryScannerSpec.scala | 22 +++++++++++ src/test/scala/tutor/utils/FileUtilSpec.scala | 20 ++++++++++ 9 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 src/main/scala/tutor/DirectoryScanner.scala create mode 100644 src/main/scala/tutor/package.scala create mode 100644 src/main/scala/tutor/utils/FileUtil.scala create mode 100644 src/test/resources/sub/SomeCode.scala create mode 100644 src/test/resources/sub/sub1/othercode.scala create mode 100644 src/test/scala/tutor/DirectoryScannerSpec.scala create mode 100644 src/test/scala/tutor/utils/FileUtilSpec.scala diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala new file mode 100644 index 0000000..53d3f16 --- /dev/null +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -0,0 +1,38 @@ +package tutor + +import java.io._ + +import tutor.utils.FileUtil +import tutor.utils.FileUtil._ + + +class DirectoryScanner { + type FileType = String + + def scan(path: Path): Map[FileType, Int] = { + val file = new File(path) + val files = file.listFiles() + files.foldLeft(Map[FileType, Int]()) { (acc, f) => + if (f.isFile) { + val fileType: FileType = FileUtil.extractExtFileName(f.getPath) + if (acc.contains(fileType)) { + acc.updated(fileType, acc(fileType) + 1) + } + else acc + (fileType -> 1) + } else { +// acc ++ scan(f.getAbsolutePath) this is wrong!!! + mergeAppend(acc, scan(f.getAbsolutePath)) + } + } + } + + def mergeAppend(map1:Map[FileType, Int], map2:Map[FileType, Int]): Map[FileType, Int] = { + map2.foldLeft(map1){ case (m1,(m2k,m2v)) => + if(m1.contains(m2k)){ + m1.updated(m2k, m1(m2k) + m2v) + }else{ + m1 + (m2k -> m2v) + } + } + } +} diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 0883083..9307179 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,10 +1,23 @@ package tutor +import java.io.File + +import tutor.utils.FileUtil.Path + object MainApp extends App{ if(args.length < 1){ println("usage: CodeAnalyzer FilePath") }else{ - val sourceCode = SourceCode.fromFile(args(0)) - println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") + val path: Path = args(0) + val file = new File(path) + if(file.isFile) { + val sourceCode = SourceCode.fromFile(path) + println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") + }else{ + val ds = new DirectoryScanner + ds.scan(path).foreach{ + case (fileType, count) => println(s"$fileType $count") + } + } } } diff --git a/src/main/scala/tutor/SourceCode.scala b/src/main/scala/tutor/SourceCode.scala index 8e99378..b92bae0 100644 --- a/src/main/scala/tutor/SourceCode.scala +++ b/src/main/scala/tutor/SourceCode.scala @@ -1,17 +1,19 @@ package tutor +import tutor.utils.FileUtil._ +import tutor.utils.FileUtil + class SourceCode(val path: String, val name: String, private val lines: List[String]) { def count: Int = lines.length } object SourceCode { - type Path = String - def fromFile(path: Path): SourceCode = { import scala.io._ val source = Source.fromFile(path) val lines = source.getLines.toList - new SourceCode(path, path.split("/").last, lines) + new SourceCode(path, FileUtil.extractLocalPath(path), lines) } + } \ No newline at end of file diff --git a/src/main/scala/tutor/package.scala b/src/main/scala/tutor/package.scala new file mode 100644 index 0000000..92f0c0c --- /dev/null +++ b/src/main/scala/tutor/package.scala @@ -0,0 +1,2 @@ +package object tutor { +} diff --git a/src/main/scala/tutor/utils/FileUtil.scala b/src/main/scala/tutor/utils/FileUtil.scala new file mode 100644 index 0000000..797b116 --- /dev/null +++ b/src/main/scala/tutor/utils/FileUtil.scala @@ -0,0 +1,17 @@ +package tutor.utils + +object FileUtil { + type Path = String + val EmptyFileType = "empty-file-type" + + def extractExtFileName(file: Path): String = { + val localPath = extractLocalPath(file) + if (localPath.contains(".")) { + localPath.split("\\.").last + } else EmptyFileType + } + + def extractLocalPath(path: Path): String = { + path.split("/").last + } +} diff --git a/src/test/resources/sub/SomeCode.scala b/src/test/resources/sub/SomeCode.scala new file mode 100644 index 0000000..01dd6b3 --- /dev/null +++ b/src/test/resources/sub/SomeCode.scala @@ -0,0 +1,16 @@ +package tutor + +class SourceCode(val path: String, val name: String, private val lines: List[String]) { + def count: Int = lines.length +} + +object SourceCode { + def fromFile(path: Path): SourceCode = { + import scala.io._ + + val source = Source.fromFile(path) + val lines = source.getLines.toList + new SourceCode(path, extractLocalPath(path), lines) + } + +} \ No newline at end of file diff --git a/src/test/resources/sub/sub1/othercode.scala b/src/test/resources/sub/sub1/othercode.scala new file mode 100644 index 0000000..01dd6b3 --- /dev/null +++ b/src/test/resources/sub/sub1/othercode.scala @@ -0,0 +1,16 @@ +package tutor + +class SourceCode(val path: String, val name: String, private val lines: List[String]) { + def count: Int = lines.length +} + +object SourceCode { + def fromFile(path: Path): SourceCode = { + import scala.io._ + + val source = Source.fromFile(path) + val lines = source.getLines.toList + new SourceCode(path, extractLocalPath(path), lines) + } + +} \ No newline at end of file diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala new file mode 100644 index 0000000..563d5fa --- /dev/null +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -0,0 +1,22 @@ +package tutor + +import org.scalatest.{FunSpec, ShouldMatchers} +import tutor.utils.FileUtil + +class DirectoryScannerSpec extends FunSpec with ShouldMatchers { + describe("DirectoryScanner") { + it("can scan directory recursively and count file numbers") { + val ds = new DirectoryScanner + ds.scan("src/test/resources") shouldBe Map((FileUtil.EmptyFileType, 1), ("scala", 2)) + } + it("can merge two maps, keep their keys and sum the values"){ + val ds = new DirectoryScanner + val map1 = Map(FileUtil.EmptyFileType -> 1, "scala" -> 1) + val map2 = Map("scala" -> 1, "xml" -> 1) + val merged = ds.mergeAppend(map1,map2) + merged should have size 3 + merged("scala") shouldBe 2 + merged("xml") shouldBe 1 + } + } +} diff --git a/src/test/scala/tutor/utils/FileUtilSpec.scala b/src/test/scala/tutor/utils/FileUtilSpec.scala new file mode 100644 index 0000000..cc15046 --- /dev/null +++ b/src/test/scala/tutor/utils/FileUtilSpec.scala @@ -0,0 +1,20 @@ +package tutor.utils + +import org.scalatest.{FunSpec, ShouldMatchers} + +class FileUtilSpec extends FunSpec with ShouldMatchers { + describe("FileUtil"){ + it("can extract file extension name"){ + val path = "src/test/build.sbt" + FileUtil.extractExtFileName(path) shouldBe "sbt" + } + it("if file has no extension name, should give EmptyFileType constant"){ + val path = "src/test/build" + FileUtil.extractExtFileName(path) shouldBe FileUtil.EmptyFileType + } + it("can extract local file path"){ + val path = "src/test/build.sbt" + FileUtil.extractLocalPath(path) shouldBe "build.sbt" + } + } +} From db42735e558f63fde67b6a717036f0e5f94f473d Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 3 Sep 2016 15:15:08 +0800 Subject: [PATCH 006/116] clean code --- src/main/scala/tutor/DirectoryScanner.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 53d3f16..aec711c 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -20,17 +20,17 @@ class DirectoryScanner { } else acc + (fileType -> 1) } else { -// acc ++ scan(f.getAbsolutePath) this is wrong!!! + // acc ++ scan(f.getAbsolutePath) this is wrong!!! mergeAppend(acc, scan(f.getAbsolutePath)) } } } - def mergeAppend(map1:Map[FileType, Int], map2:Map[FileType, Int]): Map[FileType, Int] = { - map2.foldLeft(map1){ case (m1,(m2k,m2v)) => - if(m1.contains(m2k)){ + def mergeAppend(map1: Map[FileType, Int], map2: Map[FileType, Int]): Map[FileType, Int] = { + map2.foldLeft(map1) { case (m1, (m2k, m2v)) => + if (m1.contains(m2k)) { m1.updated(m2k, m1(m2k) + m2v) - }else{ + } else { m1 + (m2k -> m2v) } } From 910601e603de621ef53043d3f1f58483694cca10 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 3 Sep 2016 15:55:11 +0800 Subject: [PATCH 007/116] change file name in test/resource to some other file name, to prevent it from interference normal sourcecode --- src/test/resources/sub/SomeCode.scala | 4 ++-- src/test/resources/sub/sub1/othercode.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/sub/SomeCode.scala b/src/test/resources/sub/SomeCode.scala index 01dd6b3..4ecea7c 100644 --- a/src/test/resources/sub/SomeCode.scala +++ b/src/test/resources/sub/SomeCode.scala @@ -1,10 +1,10 @@ package tutor -class SourceCode(val path: String, val name: String, private val lines: List[String]) { +class SomeCode(val path: String, val name: String, private val lines: List[String]) { def count: Int = lines.length } -object SourceCode { +object SomeCode { def fromFile(path: Path): SourceCode = { import scala.io._ diff --git a/src/test/resources/sub/sub1/othercode.scala b/src/test/resources/sub/sub1/othercode.scala index 01dd6b3..7926eea 100644 --- a/src/test/resources/sub/sub1/othercode.scala +++ b/src/test/resources/sub/sub1/othercode.scala @@ -1,10 +1,10 @@ package tutor -class SourceCode(val path: String, val name: String, private val lines: List[String]) { +class OtherCode(val path: String, val name: String, private val lines: List[String]) { def count: Int = lines.length } -object SourceCode { +object OtherCode { def fromFile(path: Path): SourceCode = { import scala.io._ From 6227945f11061bfc5850c5be2fb40e1a3c1d43ca Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 3 Sep 2016 16:37:31 +0800 Subject: [PATCH 008/116] remove unused package object --- src/main/scala/tutor/package.scala | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 src/main/scala/tutor/package.scala diff --git a/src/main/scala/tutor/package.scala b/src/main/scala/tutor/package.scala deleted file mode 100644 index 92f0c0c..0000000 --- a/src/main/scala/tutor/package.scala +++ /dev/null @@ -1,2 +0,0 @@ -package object tutor { -} From e6ce6dd4d82c4b11f4682772592fc66563ced9a4 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 4 Sep 2016 15:56:34 +0800 Subject: [PATCH 009/116] refactor DirectoryScanner to CodebaseAnalyzer --- src/main/scala/tutor/CodebaseAnalyzer.scala | 15 +++++++++ src/main/scala/tutor/DirectoryScanner.scala | 33 ++++--------------- src/main/scala/tutor/MainApp.scala | 4 +-- .../scala/tutor/CodebaseAnalyzerSpec.scala | 13 ++++++++ .../scala/tutor/DirectoryScannerSpec.scala | 19 ++++------- 5 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 src/main/scala/tutor/CodebaseAnalyzer.scala create mode 100644 src/test/scala/tutor/CodebaseAnalyzerSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala new file mode 100644 index 0000000..a710dfb --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -0,0 +1,15 @@ +package tutor + +import tutor.utils.FileUtil +import tutor.utils.FileUtil._ + + +trait CodebaseAnalyzer { + this: DirectoryScanner => + + type FileType = String + + def countFileNum(path: Path): Map[FileType, Int] = { + scan(path).groupBy(FileUtil.extractExtFileName).mapValues(_.length) + } +} diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index aec711c..0075dd6 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -1,37 +1,18 @@ package tutor -import java.io._ +import java.io.File -import tutor.utils.FileUtil -import tutor.utils.FileUtil._ +import tutor.utils.FileUtil.Path - -class DirectoryScanner { - type FileType = String - - def scan(path: Path): Map[FileType, Int] = { +trait DirectoryScanner { + def scan(path: Path): Seq[Path] = { val file = new File(path) val files = file.listFiles() - files.foldLeft(Map[FileType, Int]()) { (acc, f) => + files.foldLeft(Vector[Path]()) { (acc, f) => if (f.isFile) { - val fileType: FileType = FileUtil.extractExtFileName(f.getPath) - if (acc.contains(fileType)) { - acc.updated(fileType, acc(fileType) + 1) - } - else acc + (fileType -> 1) - } else { - // acc ++ scan(f.getAbsolutePath) this is wrong!!! - mergeAppend(acc, scan(f.getAbsolutePath)) - } - } - } - - def mergeAppend(map1: Map[FileType, Int], map2: Map[FileType, Int]): Map[FileType, Int] = { - map2.foldLeft(map1) { case (m1, (m2k, m2v)) => - if (m1.contains(m2k)) { - m1.updated(m2k, m1(m2k) + m2v) + acc :+ f.getAbsolutePath } else { - m1 + (m2k -> m2v) + acc ++ scan(f.getAbsolutePath) } } } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 9307179..b3d82db 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -14,8 +14,8 @@ object MainApp extends App{ val sourceCode = SourceCode.fromFile(path) println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") }else{ - val ds = new DirectoryScanner - ds.scan(path).foreach{ + val ds = new CodebaseAnalyzer with DirectoryScanner + ds.countFileNum(path).foreach{ case (fileType, count) => println(s"$fileType $count") } } diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala new file mode 100644 index 0000000..aef13d6 --- /dev/null +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -0,0 +1,13 @@ +package tutor + +import org.scalatest.{FunSpec, ShouldMatchers} +import tutor.utils.FileUtil + +class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { + describe("CodebaseAnalyzer") { + it("can scan directory recursively and count file numbers") { + val ds = new CodebaseAnalyzer with DirectoryScanner + ds.countFileNum("src/test/resources") shouldBe Map((FileUtil.EmptyFileType, 1), ("scala", 2)) + } + } +} diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index 563d5fa..92649ed 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -4,19 +4,12 @@ import org.scalatest.{FunSpec, ShouldMatchers} import tutor.utils.FileUtil class DirectoryScannerSpec extends FunSpec with ShouldMatchers { - describe("DirectoryScanner") { - it("can scan directory recursively and count file numbers") { - val ds = new DirectoryScanner - ds.scan("src/test/resources") shouldBe Map((FileUtil.EmptyFileType, 1), ("scala", 2)) - } - it("can merge two maps, keep their keys and sum the values"){ - val ds = new DirectoryScanner - val map1 = Map(FileUtil.EmptyFileType -> 1, "scala" -> 1) - val map2 = Map("scala" -> 1, "xml" -> 1) - val merged = ds.mergeAppend(map1,map2) - merged should have size 3 - merged("scala") shouldBe 2 - merged("xml") shouldBe 1 + describe("DirectoryScanner"){ + it("can scan directory recursively and return all file paths"){ + val ds = new DirectoryScanner {} + val files = ds.scan("src/test/resources") + files.length shouldBe 3 + FileUtil.extractLocalPath(files.head) shouldBe "sourceFileSample" } } } From 5a0f1fb92c9d8acdb8fe57f36d4df2ff2f6cbfb3 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 4 Sep 2016 16:23:26 +0800 Subject: [PATCH 010/116] refactor SourceCodeAnalyzer --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- src/main/scala/tutor/MainApp.scala | 6 +++--- src/main/scala/tutor/SourceCode.scala | 19 ------------------- src/main/scala/tutor/SourceCodeInfo.scala | 17 +++++++++++++++++ src/test/resources/sub/SomeCode.scala | 4 ++-- src/test/resources/sub/sub1/othercode.scala | 4 ++-- .../scala/tutor/CodebaseAnalyzerSpec.scala | 7 +++++-- .../scala/tutor/SourceCodeAnalyzerSpec.scala | 15 +++++++++++++++ src/test/scala/tutor/SourceCodeSpec.scala | 14 -------------- 9 files changed, 45 insertions(+), 43 deletions(-) delete mode 100644 src/main/scala/tutor/SourceCode.scala create mode 100644 src/main/scala/tutor/SourceCodeInfo.scala create mode 100644 src/test/scala/tutor/SourceCodeAnalyzerSpec.scala delete mode 100644 src/test/scala/tutor/SourceCodeSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index a710dfb..8dcdcab 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -5,7 +5,7 @@ import tutor.utils.FileUtil._ trait CodebaseAnalyzer { - this: DirectoryScanner => + this: DirectoryScanner with SourceCodeAnalyzer => type FileType = String diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index b3d82db..6cc8908 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -10,12 +10,12 @@ object MainApp extends App{ }else{ val path: Path = args(0) val file = new File(path) + val analyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer if(file.isFile) { - val sourceCode = SourceCode.fromFile(path) + val sourceCode = analyzer.processFile(file.getAbsolutePath) println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") }else{ - val ds = new CodebaseAnalyzer with DirectoryScanner - ds.countFileNum(path).foreach{ + analyzer.countFileNum(path).foreach{ case (fileType, count) => println(s"$fileType $count") } } diff --git a/src/main/scala/tutor/SourceCode.scala b/src/main/scala/tutor/SourceCode.scala deleted file mode 100644 index b92bae0..0000000 --- a/src/main/scala/tutor/SourceCode.scala +++ /dev/null @@ -1,19 +0,0 @@ -package tutor - -import tutor.utils.FileUtil._ -import tutor.utils.FileUtil - -class SourceCode(val path: String, val name: String, private val lines: List[String]) { - def count: Int = lines.length -} - -object SourceCode { - def fromFile(path: Path): SourceCode = { - import scala.io._ - - val source = Source.fromFile(path) - val lines = source.getLines.toList - new SourceCode(path, FileUtil.extractLocalPath(path), lines) - } - -} \ No newline at end of file diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala new file mode 100644 index 0000000..b8448e2 --- /dev/null +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -0,0 +1,17 @@ +package tutor + +import tutor.utils.FileUtil._ +import tutor.utils.FileUtil + +case class SourceCodeInfo(path: String, name: String, count: Int) + +trait SourceCodeAnalyzer { + def processFile(path: Path): SourceCodeInfo = { + import scala.io._ + + val source = Source.fromFile(path) + val lines = source.getLines.toList + SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) + } + +} \ No newline at end of file diff --git a/src/test/resources/sub/SomeCode.scala b/src/test/resources/sub/SomeCode.scala index 4ecea7c..17623d4 100644 --- a/src/test/resources/sub/SomeCode.scala +++ b/src/test/resources/sub/SomeCode.scala @@ -5,12 +5,12 @@ class SomeCode(val path: String, val name: String, private val lines: List[Strin } object SomeCode { - def fromFile(path: Path): SourceCode = { + def fromFile(path: Path): SourceCodeInfo = { import scala.io._ val source = Source.fromFile(path) val lines = source.getLines.toList - new SourceCode(path, extractLocalPath(path), lines) + new SourceCodeInfo(path, extractLocalPath(path), lines) } } \ No newline at end of file diff --git a/src/test/resources/sub/sub1/othercode.scala b/src/test/resources/sub/sub1/othercode.scala index 7926eea..f448649 100644 --- a/src/test/resources/sub/sub1/othercode.scala +++ b/src/test/resources/sub/sub1/othercode.scala @@ -5,12 +5,12 @@ class OtherCode(val path: String, val name: String, private val lines: List[Stri } object OtherCode { - def fromFile(path: Path): SourceCode = { + def fromFile(path: Path): SourceCodeInfo = { import scala.io._ val source = Source.fromFile(path) val lines = source.getLines.toList - new SourceCode(path, extractLocalPath(path), lines) + new SourceCodeInfo(path, extractLocalPath(path), lines) } } \ No newline at end of file diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index aef13d6..2c98e29 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -2,12 +2,15 @@ package tutor import org.scalatest.{FunSpec, ShouldMatchers} import tutor.utils.FileUtil +import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { describe("CodebaseAnalyzer") { it("can scan directory recursively and count file numbers") { - val ds = new CodebaseAnalyzer with DirectoryScanner - ds.countFileNum("src/test/resources") shouldBe Map((FileUtil.EmptyFileType, 1), ("scala", 2)) + val ds = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer{ + override def scan(path: Path): Seq[Path] = List("a.scala","b.scala","c.sbt","d") + } + ds.countFileNum("any path") should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1),("sbt",1)) } } } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala new file mode 100644 index 0000000..08e5923 --- /dev/null +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -0,0 +1,15 @@ +package tutor + +import org.scalatest.{FunSpec, ShouldMatchers} + +class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers{ + describe("SourceCode object"){ + it("can read file and create a SourceCode instance"){ + val sca = new SourceCodeAnalyzer {} + val sourceCodeInfo = sca.processFile("./src/test/resources/sourceFileSample") + sourceCodeInfo.name shouldBe "sourceFileSample" + sourceCodeInfo.path shouldBe "./src/test/resources/sourceFileSample" + sourceCodeInfo.count shouldBe 108 + } + } +} diff --git a/src/test/scala/tutor/SourceCodeSpec.scala b/src/test/scala/tutor/SourceCodeSpec.scala deleted file mode 100644 index e3c56aa..0000000 --- a/src/test/scala/tutor/SourceCodeSpec.scala +++ /dev/null @@ -1,14 +0,0 @@ -package tutor - -import org.scalatest.{FunSpec, ShouldMatchers} - -class SourceCodeSpec extends FunSpec with ShouldMatchers{ - describe("SourceCode object"){ - it("can read file and create a SourceCode instance"){ - val sourceCode = SourceCode.fromFile("./src/test/resources/sourceFileSample") - sourceCode.name shouldBe "sourceFileSample" - sourceCode.path shouldBe "./src/test/resources/sourceFileSample" - sourceCode.count shouldBe 108 - } - } -} From dfdc087fa53009c02217cbe9e1f42eab6ac009c3 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 10:14:43 +0800 Subject: [PATCH 011/116] refactor CodebaseAnalyzer --- src/main/scala/tutor/CodebaseAnalyzer.scala | 6 +++++- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 14 ++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 8dcdcab..27dd2f1 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -10,6 +10,10 @@ trait CodebaseAnalyzer { type FileType = String def countFileNum(path: Path): Map[FileType, Int] = { - scan(path).groupBy(FileUtil.extractExtFileName).mapValues(_.length) + countFileTypeNum(scan(path)) + } + + private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { + files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) } } diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 2c98e29..356b42a 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -5,12 +5,18 @@ import tutor.utils.FileUtil import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { + + val ds = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { + override def scan(path: Path): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") + } + describe("CodebaseAnalyzer") { it("can scan directory recursively and count file numbers") { - val ds = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer{ - override def scan(path: Path): Seq[Path] = List("a.scala","b.scala","c.sbt","d") - } - ds.countFileNum("any path") should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1),("sbt",1)) + ds.countFileNum("any path") should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + } + it("can count file numbers by type") { + val ls = List("a.scala", "b.scala", "c.sbt", "d") + ds.countFileTypeNum(ls) should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } } } From 749dda8395fcba863301b0ad59599d5822ddbb23 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 10:40:17 +0800 Subject: [PATCH 012/116] refactor MainApp --- src/main/scala/tutor/CodebaseAnalyzer.scala | 9 ++++--- src/main/scala/tutor/MainApp.scala | 19 +++++++-------- src/main/scala/tutor/ReportFormatter.scala | 13 ++++++++++ .../scala/tutor/CodebaseAnalyzerSpec.scala | 5 +--- .../scala/tutor/ReportFormatterSpec.scala | 24 +++++++++++++++++++ 5 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 src/main/scala/tutor/ReportFormatter.scala create mode 100644 src/test/scala/tutor/ReportFormatterSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 27dd2f1..1c674d4 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,17 +3,16 @@ package tutor import tutor.utils.FileUtil import tutor.utils.FileUtil._ +case class CodebaseInfo(fileTypeNums: Map[String, Int]) trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => - type FileType = String - - def countFileNum(path: Path): Map[FileType, Int] = { + def countFileNum(path: Path): CodebaseInfo = { countFileTypeNum(scan(path)) } - private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { - files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) + private[tutor] def countFileTypeNum(files: Seq[Path]): CodebaseInfo = { + CodebaseInfo(files.groupBy(FileUtil.extractExtFileName).mapValues(_.length)) } } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 6cc8908..e66774f 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -4,20 +4,19 @@ import java.io.File import tutor.utils.FileUtil.Path -object MainApp extends App{ - if(args.length < 1){ +object MainApp extends App with ReportFormatter { + if (args.length < 1) { println("usage: CodeAnalyzer FilePath") - }else{ + } else { val path: Path = args(0) val file = new File(path) val analyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer - if(file.isFile) { - val sourceCode = analyzer.processFile(file.getAbsolutePath) - println(s"name: ${sourceCode.name} lines: ${sourceCode.count}") - }else{ - analyzer.countFileNum(path).foreach{ - case (fileType, count) => println(s"$fileType $count") - } + val rs = if (file.isFile) { + format(analyzer.processFile(file.getAbsolutePath)) + } else { + format(analyzer.countFileNum(path)) } + println(rs) } + } diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala new file mode 100644 index 0000000..4ad957e --- /dev/null +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -0,0 +1,13 @@ +package tutor + +trait ReportFormatter { + def format(codebaseInfo: CodebaseInfo): String = { + codebaseInfo.fileTypeNums.map { + case (fileType, count) => s"$fileType $count" + }.mkString("\n") + } + + def format(sourceCode: SourceCodeInfo): String = { + s"name: ${sourceCode.name} lines: ${sourceCode.count}" + } +} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 356b42a..2c9379e 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -11,12 +11,9 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { } describe("CodebaseAnalyzer") { - it("can scan directory recursively and count file numbers") { - ds.countFileNum("any path") should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) - } it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") - ds.countFileTypeNum(ls) should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + ds.countFileTypeNum(ls).fileTypeNums should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } } } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala new file mode 100644 index 0000000..3119d91 --- /dev/null +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -0,0 +1,24 @@ +package tutor + +import org.scalatest.{FunSpec, ShouldMatchers} +import tutor.utils.FileUtil + +class ReportFormatterSpec extends FunSpec with ShouldMatchers{ + + val rf = new ReportFormatter {} + + describe("ReportFormatter"){ + it("can format SourceCodeInfo"){ + rf.format(SourceCodeInfo("somepath","some name", 10)) shouldBe "name: some name lines: 10" + } + it("can format CodebaseInfo"){ + val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1)) + rf.format(codebaseInfo) shouldBe + """ + |sbt 1 + |scala 2 + |empty-file-type 1 + """.trim.stripMargin + } + } +} From 5da93b373397c5d3bcb9fdb006e5f016f25e5a70 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 15:28:49 +0800 Subject: [PATCH 013/116] refactor MainApp --- src/main/scala/tutor/CodebaseAnalyzer.scala | 13 ++++++++----- src/main/scala/tutor/MainApp.scala | 2 +- src/main/scala/tutor/ReportFormatter.scala | 9 +++++++-- src/main/scala/tutor/SourceCodeInfo.scala | 2 +- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 12 +++++++++++- src/test/scala/tutor/ReportFormatterSpec.scala | 5 ++++- src/test/scala/tutor/SourceCodeAnalyzerSpec.scala | 2 +- 7 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 1c674d4..0c01e76 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,16 +3,19 @@ package tutor import tutor.utils.FileUtil import tutor.utils.FileUtil._ -case class CodebaseInfo(fileTypeNums: Map[String, Int]) +case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double) trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => - def countFileNum(path: Path): CodebaseInfo = { - countFileTypeNum(scan(path)) + def analyze(path: Path): CodebaseInfo = { + val files = scan(path) + val avgLineCount = files.map(processFile).map(_.count).sum.toDouble / files.length + CodebaseInfo(countFileTypeNum(files), avgLineCount) } - private[tutor] def countFileTypeNum(files: Seq[Path]): CodebaseInfo = { - CodebaseInfo(files.groupBy(FileUtil.extractExtFileName).mapValues(_.length)) + private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { + files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) } + } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index e66774f..d82f6c9 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -14,7 +14,7 @@ object MainApp extends App with ReportFormatter { val rs = if (file.isFile) { format(analyzer.processFile(file.getAbsolutePath)) } else { - format(analyzer.countFileNum(path)) + format(analyzer.analyze(path)) } println(rs) } diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index 4ad957e..864a694 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -1,13 +1,18 @@ package tutor trait ReportFormatter { + val separator = "---------------------------" def format(codebaseInfo: CodebaseInfo): String = { codebaseInfo.fileTypeNums.map { case (fileType, count) => s"$fileType $count" - }.mkString("\n") + }.mkString("\n") ++ + "\n\n" ++ + separator ++ "\n" ++ + s"avg line count: ${codebaseInfo.avgLineCount}" + } def format(sourceCode: SourceCodeInfo): String = { - s"name: ${sourceCode.name} lines: ${sourceCode.count}" + s"name: ${sourceCode.localPath} lines: ${sourceCode.count}" } } diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index b8448e2..06b11dc 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil._ import tutor.utils.FileUtil -case class SourceCodeInfo(path: String, name: String, count: Int) +case class SourceCodeInfo(path: String, localPath: String, count: Int) trait SourceCodeAnalyzer { def processFile(path: Path): SourceCodeInfo = { diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 2c9379e..e0aca3d 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -8,12 +8,22 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val ds = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") + + override def processFile(path: Path): SourceCodeInfo = path match { + case "a.scala" => SourceCodeInfo(path, path,10) + case "b.scala" => SourceCodeInfo(path, path,10) + case "c.sbt" => SourceCodeInfo(path, path,5) + case "d" => SourceCodeInfo(path, path,5) + } } describe("CodebaseAnalyzer") { it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") - ds.countFileTypeNum(ls).fileTypeNums should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + ds.countFileTypeNum(ls) should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + } + it("can analyze avg file count"){ + ds.analyze("anypath").avgLineCount shouldBe 7.5 } } } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 3119d91..66fd65c 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -12,12 +12,15 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers{ rf.format(SourceCodeInfo("somepath","some name", 10)) shouldBe "name: some name lines: 10" } it("can format CodebaseInfo"){ - val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1)) + val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1),7.5) rf.format(codebaseInfo) shouldBe """ |sbt 1 |scala 2 |empty-file-type 1 + | + |--------------------------- + |avg line count: 7.5 """.trim.stripMargin } } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 08e5923..79dd901 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -7,7 +7,7 @@ class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers{ it("can read file and create a SourceCode instance"){ val sca = new SourceCodeAnalyzer {} val sourceCodeInfo = sca.processFile("./src/test/resources/sourceFileSample") - sourceCodeInfo.name shouldBe "sourceFileSample" + sourceCodeInfo.localPath shouldBe "sourceFileSample" sourceCodeInfo.path shouldBe "./src/test/resources/sourceFileSample" sourceCodeInfo.count shouldBe 108 } From 5e281ac2932ede7ecaaacfe38125754de55fedcd Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 15:45:11 +0800 Subject: [PATCH 014/116] complete longest file analyze feature --- src/main/scala/tutor/CodebaseAnalyzer.scala | 11 ++++++++--- src/main/scala/tutor/ReportFormatter.scala | 5 +++-- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 6 ++++++ src/test/scala/tutor/ReportFormatterSpec.scala | 6 +++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 0c01e76..37a9c9d 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,19 +3,24 @@ package tutor import tutor.utils.FileUtil import tutor.utils.FileUtil._ -case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double) +case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double, longestFileInfo: SourceCodeInfo) trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => def analyze(path: Path): CodebaseInfo = { val files = scan(path) - val avgLineCount = files.map(processFile).map(_.count).sum.toDouble / files.length - CodebaseInfo(countFileTypeNum(files), avgLineCount) + val sourceCodeInfoes: Seq[SourceCodeInfo] = files.map(processFile) + val avgLineCount = sourceCodeInfoes.map(_.count).sum.toDouble / files.length + CodebaseInfo(countFileTypeNum(files), avgLineCount, null) } private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) } + private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { + sourceCodeInfos.sortBy(_.count).last + } + } diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index 864a694..496b0d4 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -3,13 +3,14 @@ package tutor trait ReportFormatter { val separator = "---------------------------" def format(codebaseInfo: CodebaseInfo): String = { + val longestFileInfo: SourceCodeInfo = codebaseInfo.longestFileInfo codebaseInfo.fileTypeNums.map { case (fileType, count) => s"$fileType $count" }.mkString("\n") ++ "\n\n" ++ separator ++ "\n" ++ - s"avg line count: ${codebaseInfo.avgLineCount}" - + s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ + s"longest file: ${longestFileInfo.localPath} ${longestFileInfo.count}" } def format(sourceCode: SourceCodeInfo): String = { diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index e0aca3d..99e841e 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -25,5 +25,11 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { it("can analyze avg file count"){ ds.analyze("anypath").avgLineCount shouldBe 7.5 } + it("can find longest file"){ + val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) + val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) + ds.longestFile(List(aInfo,bInfo)) shouldBe aInfo + } } + } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 66fd65c..9887ac5 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -12,7 +12,10 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers{ rf.format(SourceCodeInfo("somepath","some name", 10)) shouldBe "name: some name lines: 10" } it("can format CodebaseInfo"){ - val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1),7.5) + val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), + 7.5 + , SourceCodeInfo("a.scala","a.scala", 10) + ) rf.format(codebaseInfo) shouldBe """ |sbt 1 @@ -21,6 +24,7 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers{ | |--------------------------- |avg line count: 7.5 + |longest file: a.scala 10 """.trim.stripMargin } } From 63778ef14e9462331a29a56f8ac616f45fbc8f79 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 16:30:10 +0800 Subject: [PATCH 015/116] complete top10 long files analyze feature --- src/main/scala/tutor/CodebaseAnalyzer.scala | 10 ++++- src/main/scala/tutor/ReportFormatter.scala | 19 ++++++--- .../scala/tutor/CodebaseAnalyzerSpec.scala | 21 ++++++---- .../scala/tutor/ReportFormatterSpec.scala | 42 ++++++++++++------- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 37a9c9d..969d208 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,7 +3,10 @@ package tutor import tutor.utils.FileUtil import tutor.utils.FileUtil._ -case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double, longestFileInfo: SourceCodeInfo) +case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double, + longestFileInfo: SourceCodeInfo, + top10Files: Seq[SourceCodeInfo] + ) trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => @@ -12,7 +15,7 @@ trait CodebaseAnalyzer { val files = scan(path) val sourceCodeInfoes: Seq[SourceCodeInfo] = files.map(processFile) val avgLineCount = sourceCodeInfoes.map(_.count).sum.toDouble / files.length - CodebaseInfo(countFileTypeNum(files), avgLineCount, null) + CodebaseInfo(countFileTypeNum(files), avgLineCount, longestFile(sourceCodeInfoes), top10Files(sourceCodeInfoes)) } private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { @@ -23,4 +26,7 @@ trait CodebaseAnalyzer { sourceCodeInfos.sortBy(_.count).last } + private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { + sourceCodeInfos.sortBy(_.count).reverse.take(10) + } } diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index 496b0d4..368edc1 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -1,19 +1,28 @@ package tutor trait ReportFormatter { - val separator = "---------------------------" def format(codebaseInfo: CodebaseInfo): String = { val longestFileInfo: SourceCodeInfo = codebaseInfo.longestFileInfo codebaseInfo.fileTypeNums.map { case (fileType, count) => s"$fileType $count" }.mkString("\n") ++ - "\n\n" ++ - separator ++ "\n" ++ - s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ - s"longest file: ${longestFileInfo.localPath} ${longestFileInfo.count}" + "\n" ++ + ReportFormatter.separator ++ "\n\n" ++ + s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ + s"longest file: ${longestFileInfo.localPath} ${longestFileInfo.count}" ++ + "\n" ++ + ReportFormatter.separator ++ "\n\n" ++ + "top 10 long files\n" ++ + codebaseInfo.top10Files.map { + s => s"${s.localPath} ${s.count}" + }.mkString("\n") } def format(sourceCode: SourceCodeInfo): String = { s"name: ${sourceCode.localPath} lines: ${sourceCode.count}" } } + +object ReportFormatter { + val separator = "---------------------------" +} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 99e841e..e529600 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -10,10 +10,10 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { override def scan(path: Path): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): SourceCodeInfo = path match { - case "a.scala" => SourceCodeInfo(path, path,10) - case "b.scala" => SourceCodeInfo(path, path,10) - case "c.sbt" => SourceCodeInfo(path, path,5) - case "d" => SourceCodeInfo(path, path,5) + case "a.scala" => SourceCodeInfo(path, path, 10) + case "b.scala" => SourceCodeInfo(path, path, 10) + case "c.sbt" => SourceCodeInfo(path, path, 5) + case "d" => SourceCodeInfo(path, path, 5) } } @@ -22,14 +22,19 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val ls = List("a.scala", "b.scala", "c.sbt", "d") ds.countFileTypeNum(ls) should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } - it("can analyze avg file count"){ + it("can analyze avg file count") { ds.analyze("anypath").avgLineCount shouldBe 7.5 } - it("can find longest file"){ + it("can find longest file") { val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) - ds.longestFile(List(aInfo,bInfo)) shouldBe aInfo + ds.longestFile(List(aInfo, bInfo)) shouldBe aInfo + } + it("will return top 10 longest files"){ + val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) + val top10LongFiles = ds.top10Files(sourceCodeInfos) + top10LongFiles should have size 10 + top10LongFiles should not contain SourceCodeInfo("1.scala","1.scala",1) } } - } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 9887ac5..55d4b41 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -3,28 +3,42 @@ package tutor import org.scalatest.{FunSpec, ShouldMatchers} import tutor.utils.FileUtil -class ReportFormatterSpec extends FunSpec with ShouldMatchers{ +class ReportFormatterSpec extends FunSpec with ShouldMatchers { val rf = new ReportFormatter {} - describe("ReportFormatter"){ - it("can format SourceCodeInfo"){ - rf.format(SourceCodeInfo("somepath","some name", 10)) shouldBe "name: some name lines: 10" + describe("ReportFormatter") { + it("can format SourceCodeInfo") { + rf.format(SourceCodeInfo("somepath", "some name", 10)) shouldBe "name: some name lines: 10" } - it("can format CodebaseInfo"){ + it("can format CodebaseInfo") { val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), 7.5 - , SourceCodeInfo("a.scala","a.scala", 10) + , SourceCodeInfo("a.scala", "a.scala", 10) + , {for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i)} ) rf.format(codebaseInfo) shouldBe - """ - |sbt 1 - |scala 2 - |empty-file-type 1 - | - |--------------------------- - |avg line count: 7.5 - |longest file: a.scala 10 + s""" + |sbt 1 + |scala 2 + |empty-file-type 1 + |${ReportFormatter.separator} + | + |avg line count: 7.5 + |longest file: a.scala 10 + |${ReportFormatter.separator} + | + |top 10 long files + |10.scala 10 + |9.scala 9 + |8.scala 8 + |7.scala 7 + |6.scala 6 + |5.scala 5 + |4.scala 4 + |3.scala 3 + |2.scala 2 + |1.scala 1 """.trim.stripMargin } } From 2d215ad3e94470264afba9fd72631a6bf519fedc Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 16:38:39 +0800 Subject: [PATCH 016/116] complete total line num count feature --- src/main/scala/tutor/CodebaseAnalyzer.scala | 8 ++++++-- src/main/scala/tutor/ReportFormatter.scala | 1 + src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 9 +++++++-- src/test/scala/tutor/ReportFormatterSpec.scala | 9 ++++++--- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 969d208..738f1e5 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil import tutor.utils.FileUtil._ -case class CodebaseInfo(fileTypeNums: Map[String, Int], avgLineCount: Double, +case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avgLineCount: Double, longestFileInfo: SourceCodeInfo, top10Files: Seq[SourceCodeInfo] ) @@ -15,7 +15,7 @@ trait CodebaseAnalyzer { val files = scan(path) val sourceCodeInfoes: Seq[SourceCodeInfo] = files.map(processFile) val avgLineCount = sourceCodeInfoes.map(_.count).sum.toDouble / files.length - CodebaseInfo(countFileTypeNum(files), avgLineCount, longestFile(sourceCodeInfoes), top10Files(sourceCodeInfoes)) + CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfoes),avgLineCount, longestFile(sourceCodeInfoes), top10Files(sourceCodeInfoes)) } private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { @@ -29,4 +29,8 @@ trait CodebaseAnalyzer { private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { sourceCodeInfos.sortBy(_.count).reverse.take(10) } + + private[tutor] def totalLineCount(sourceCodeInfos: Seq[SourceCodeInfo]): Int = { + sourceCodeInfos.map(_.count).sum + } } diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index 368edc1..72b9a79 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -8,6 +8,7 @@ trait ReportFormatter { }.mkString("\n") ++ "\n" ++ ReportFormatter.separator ++ "\n\n" ++ + s"total line count: ${codebaseInfo.totalLineCount}" ++ "\n" ++ s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ s"longest file: ${longestFileInfo.localPath} ${longestFileInfo.count}" ++ "\n" ++ diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index e529600..15c8abb 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -17,6 +17,9 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { } } + val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) + val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) + describe("CodebaseAnalyzer") { it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") @@ -26,8 +29,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { ds.analyze("anypath").avgLineCount shouldBe 7.5 } it("can find longest file") { - val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) - val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) + ds.longestFile(List(aInfo, bInfo)) shouldBe aInfo } it("will return top 10 longest files"){ @@ -36,5 +38,8 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { top10LongFiles should have size 10 top10LongFiles should not contain SourceCodeInfo("1.scala","1.scala",1) } + it("can count total line numbers"){ + ds.totalLineCount(List(aInfo, bInfo)) shouldBe 15 + } } } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 55d4b41..9b85cee 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -13,9 +13,11 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { } it("can format CodebaseInfo") { val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), - 7.5 - , SourceCodeInfo("a.scala", "a.scala", 10) - , {for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i)} + totalLineCount = 15, + avgLineCount = 7.5, + SourceCodeInfo("a.scala", "a.scala", 10), { + for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) + } ) rf.format(codebaseInfo) shouldBe s""" @@ -24,6 +26,7 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { |empty-file-type 1 |${ReportFormatter.separator} | + |total line count: 15 |avg line count: 7.5 |longest file: a.scala 10 |${ReportFormatter.separator} From 061e1622be127c9e65d6783d61c7f5c7270cfc61 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 17:20:19 +0800 Subject: [PATCH 017/116] using loan pattern --- src/main/scala/tutor/MainApp.scala | 17 +++++++++++++---- src/main/scala/tutor/utils/WriteSupport.scala | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 src/main/scala/tutor/utils/WriteSupport.scala diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index d82f6c9..73e1422 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,12 +1,13 @@ package tutor -import java.io.File +import java.io.{BufferedWriter, File, FileWriter, Writer} import tutor.utils.FileUtil.Path +import tutor.utils.WriteSupport -object MainApp extends App with ReportFormatter { +object MainApp extends App with ReportFormatter with WriteSupport { if (args.length < 1) { - println("usage: CodeAnalyzer FilePath") + println("usage: CodeAnalyzer FilePath [-oOutputfile]") } else { val path: Path = args(0) val file = new File(path) @@ -16,7 +17,15 @@ object MainApp extends App with ReportFormatter { } else { format(analyzer.analyze(path)) } - println(rs) + if (args.length > 1) { + val output = args(1).drop(2) + withWriter(output) { + _.write(rs) + } + println(s"report saved into $output") + } else { + println(rs) + } } } diff --git a/src/main/scala/tutor/utils/WriteSupport.scala b/src/main/scala/tutor/utils/WriteSupport.scala new file mode 100644 index 0000000..074e54b --- /dev/null +++ b/src/main/scala/tutor/utils/WriteSupport.scala @@ -0,0 +1,19 @@ +package tutor.utils + +import java.io.{BufferedWriter, File, FileWriter, Writer} + +trait WriteSupport { + + def withWriter(path: String)(f: Writer => Unit): Unit ={ + var writer: Writer = null + try { + val file = new File(path) + if (!file.exists()) file.createNewFile() + writer = new BufferedWriter(new FileWriter(file)) + f(writer) + writer.flush() + } finally { + if (writer != null) writer.close() + } + } +} From 87b730e94506e85d3b0dcdf833149489d0ba685a Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 17:39:17 +0800 Subject: [PATCH 018/116] clean code --- src/main/scala/tutor/MainApp.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 73e1422..fec39b2 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,6 +1,6 @@ package tutor -import java.io.{BufferedWriter, File, FileWriter, Writer} +import java.io.File import tutor.utils.FileUtil.Path import tutor.utils.WriteSupport From f5edb1e7bef211bf291d56e6bbce24d6ff81f582 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 11 Sep 2016 22:15:31 +0800 Subject: [PATCH 019/116] implementing ordering typeclass --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- src/main/scala/tutor/SourceCodeInfo.scala | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 738f1e5..949979c 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -23,7 +23,7 @@ trait CodebaseAnalyzer { } private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { - sourceCodeInfos.sortBy(_.count).last + sourceCodeInfos.sorted.last } private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index 06b11dc..68a0353 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -4,6 +4,11 @@ import tutor.utils.FileUtil._ import tutor.utils.FileUtil case class SourceCodeInfo(path: String, localPath: String, count: Int) +object SourceCodeInfo{ + implicit object SourceCodeInfoOrdering extends Ordering[SourceCodeInfo] { + override def compare(x: SourceCodeInfo, y: SourceCodeInfo): Int = x.count compare y.count + } +} trait SourceCodeAnalyzer { def processFile(path: Path): SourceCodeInfo = { From 833ddf4f7f26b98ed78137608e73bf2c38ce3e4f Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 23 Dec 2016 14:37:49 +0800 Subject: [PATCH 020/116] add type definition to make intellij happy --- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 15c8abb..e6111d3 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -23,7 +23,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { describe("CodebaseAnalyzer") { it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") - ds.countFileTypeNum(ls) should contain theSameElementsAs Map(("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + ds.countFileTypeNum(ls) should contain theSameElementsAs Map[String,Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } it("can analyze avg file count") { ds.analyze("anypath").avgLineCount shouldBe 7.5 From eace62e377ac023d3e88de7c121ac9aa6d1c4779 Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 23 Dec 2016 14:41:51 +0800 Subject: [PATCH 021/116] remove 2 demo files --- src/main/scala/lesson1/hello/Main.scala | 7 ------- src/test/scala/lesson1/hello/MainTest.scala | 11 ----------- 2 files changed, 18 deletions(-) delete mode 100644 src/main/scala/lesson1/hello/Main.scala delete mode 100644 src/test/scala/lesson1/hello/MainTest.scala diff --git a/src/main/scala/lesson1/hello/Main.scala b/src/main/scala/lesson1/hello/Main.scala deleted file mode 100644 index a81ee3a..0000000 --- a/src/main/scala/lesson1/hello/Main.scala +++ /dev/null @@ -1,7 +0,0 @@ -package lesson1.hello - -object Main extends App { - println("hello,world") - - def add(x:Int, y:Int):Int = x + y -} diff --git a/src/test/scala/lesson1/hello/MainTest.scala b/src/test/scala/lesson1/hello/MainTest.scala deleted file mode 100644 index 64dcbb3..0000000 --- a/src/test/scala/lesson1/hello/MainTest.scala +++ /dev/null @@ -1,11 +0,0 @@ -package lesson1.hello - -import org.scalatest.{FunSpec, ShouldMatchers} - -class MainTest extends FunSpec with ShouldMatchers{ - describe("Main"){ - it("can add x and y"){ - Main.add(1,2) shouldBe 3 - } - } -} From 20c2629f64e9084500fa9e18e0b369cff7d20b36 Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 23 Dec 2016 15:01:35 +0800 Subject: [PATCH 022/116] refactor code --- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index e6111d3..8385261 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -6,7 +6,7 @@ import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { - val ds = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { + val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): SourceCodeInfo = path match { @@ -23,23 +23,23 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { describe("CodebaseAnalyzer") { it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") - ds.countFileTypeNum(ls) should contain theSameElementsAs Map[String,Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String,Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } it("can analyze avg file count") { - ds.analyze("anypath").avgLineCount shouldBe 7.5 + codeBaseAnalyzer.analyze("anypath").avgLineCount shouldBe 7.5 } it("can find longest file") { - ds.longestFile(List(aInfo, bInfo)) shouldBe aInfo + codeBaseAnalyzer.longestFile(List(aInfo, bInfo)) shouldBe aInfo } it("will return top 10 longest files"){ val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) - val top10LongFiles = ds.top10Files(sourceCodeInfos) + val top10LongFiles = codeBaseAnalyzer.top10Files(sourceCodeInfos) top10LongFiles should have size 10 top10LongFiles should not contain SourceCodeInfo("1.scala","1.scala",1) } it("can count total line numbers"){ - ds.totalLineCount(List(aInfo, bInfo)) shouldBe 15 + codeBaseAnalyzer.totalLineCount(List(aInfo, bInfo)) shouldBe 15 } } } From 1f2dbb456335c7690aa0b3e28cf7bb551be1fa9b Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 23 Dec 2016 16:04:07 +0800 Subject: [PATCH 023/116] fix typo --- src/main/scala/tutor/CodebaseAnalyzer.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 949979c..22e345f 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -13,9 +13,9 @@ trait CodebaseAnalyzer { def analyze(path: Path): CodebaseInfo = { val files = scan(path) - val sourceCodeInfoes: Seq[SourceCodeInfo] = files.map(processFile) - val avgLineCount = sourceCodeInfoes.map(_.count).sum.toDouble / files.length - CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfoes),avgLineCount, longestFile(sourceCodeInfoes), top10Files(sourceCodeInfoes)) + val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile) + val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos),avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos)) } private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { From fedec8b062a416e9e1455d5fa633073175c118da Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 15:33:24 +0800 Subject: [PATCH 024/116] output some log info --- src/main/scala/tutor/DirectoryScanner.scala | 4 +++- src/main/scala/tutor/SourceCodeInfo.scala | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 0075dd6..619e4af 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -2,10 +2,12 @@ package tutor import java.io.File +import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil.Path -trait DirectoryScanner { +trait DirectoryScanner extends StrictLogging { def scan(path: Path): Seq[Path] = { + logger.info(s"scanning $path") val file = new File(path) val files = file.listFiles() files.foldLeft(Vector[Path]()) { (acc, f) => diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index 68a0353..c94c047 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -1,5 +1,6 @@ package tutor +import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil._ import tutor.utils.FileUtil @@ -10,10 +11,10 @@ object SourceCodeInfo{ } } -trait SourceCodeAnalyzer { +trait SourceCodeAnalyzer extends StrictLogging{ def processFile(path: Path): SourceCodeInfo = { import scala.io._ - + logger.info(s"processing $path") val source = Source.fromFile(path) val lines = source.getLines.toList SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) From f6fb61a2c1b83c6564c15e48cdf11894eb9ea14c Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 16:22:49 +0800 Subject: [PATCH 025/116] do a big refactoring to make it only accept given file types --- src/main/scala/tutor/CodebaseAnalyzer.scala | 4 +-- src/main/scala/tutor/DirectoryScanner.scala | 26 ++++++++++++++++--- src/main/scala/tutor/KnowFileTypes.scala | 6 +++++ src/main/scala/tutor/MainApp.scala | 2 +- src/test/resources/sub/sub1/ignore.me | 1 + .../sub1/{othercode.scala => othercode.java} | 0 .../scala/tutor/CodebaseAnalyzerSpec.scala | 4 +-- .../scala/tutor/DirectoryScannerSpec.scala | 9 ++++--- 8 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 src/main/scala/tutor/KnowFileTypes.scala create mode 100644 src/test/resources/sub/sub1/ignore.me rename src/test/resources/sub/sub1/{othercode.scala => othercode.java} (100%) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 22e345f..039ce95 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -11,8 +11,8 @@ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avg trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => - def analyze(path: Path): CodebaseInfo = { - val files = scan(path) + def analyze(path: Path, knownFileTypes: Set[String]): CodebaseInfo = { + val files = scan(path,knownFileTypes) val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile) val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos),avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos)) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 619e4af..2928588 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -3,19 +3,37 @@ package tutor import java.io.File import com.typesafe.scalalogging.slf4j.StrictLogging +import tutor.utils.FileUtil import tutor.utils.FileUtil.Path trait DirectoryScanner extends StrictLogging { - def scan(path: Path): Seq[Path] = { - logger.info(s"scanning $path") + /** + * recursively scan given directory, get all file path whose ext is in knownFileTypes Set + * + * @param path + * @param knownFileTypes

file ext, like scala, java etc.

+ * @return + */ + def scan(path: Path, knownFileTypes: Set[String]): Seq[Path] = { + logger.info(s"scanning $path for known file types $knownFileTypes") val file = new File(path) val files = file.listFiles() files.foldLeft(Vector[Path]()) { (acc, f) => if (f.isFile) { - acc :+ f.getAbsolutePath + if(shouldAccept(f.getPath, knownFileTypes)) { + logger.info(s"add file to fold ${f.getAbsolutePath}") + acc :+ f.getAbsolutePath + }else{ + acc + } } else { - acc ++ scan(f.getAbsolutePath) + acc ++ scan(f.getAbsolutePath, knownFileTypes) } } } + + private def shouldAccept(path: Path, knownFileTypes: Set[String]): Boolean = { + logger.info(s"check if should accept $path") + knownFileTypes.contains(FileUtil.extractExtFileName(path)) + } } diff --git a/src/main/scala/tutor/KnowFileTypes.scala b/src/main/scala/tutor/KnowFileTypes.scala new file mode 100644 index 0000000..ede18e2 --- /dev/null +++ b/src/main/scala/tutor/KnowFileTypes.scala @@ -0,0 +1,6 @@ +package tutor + +object KnowFileTypes { + val knownFileTypes: Set[String] = + Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp") +} diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index fec39b2..ca55bb0 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,7 +15,7 @@ object MainApp extends App with ReportFormatter with WriteSupport { val rs = if (file.isFile) { format(analyzer.processFile(file.getAbsolutePath)) } else { - format(analyzer.analyze(path)) + format(analyzer.analyze(path, KnowFileTypes.knownFileTypes)) } if (args.length > 1) { val output = args(1).drop(2) diff --git a/src/test/resources/sub/sub1/ignore.me b/src/test/resources/sub/sub1/ignore.me new file mode 100644 index 0000000..8a8a0cd --- /dev/null +++ b/src/test/resources/sub/sub1/ignore.me @@ -0,0 +1 @@ +this file should be ignored by directory scanner \ No newline at end of file diff --git a/src/test/resources/sub/sub1/othercode.scala b/src/test/resources/sub/sub1/othercode.java similarity index 100% rename from src/test/resources/sub/sub1/othercode.scala rename to src/test/resources/sub/sub1/othercode.java diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 8385261..a3b07c3 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -7,7 +7,7 @@ import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { - override def scan(path: Path): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") + override def scan(path: Path,knowFileTypes: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): SourceCodeInfo = path match { case "a.scala" => SourceCodeInfo(path, path, 10) @@ -26,7 +26,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String,Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } it("can analyze avg file count") { - codeBaseAnalyzer.analyze("anypath").avgLineCount shouldBe 7.5 + codeBaseAnalyzer.analyze("anypath",KnowFileTypes.knownFileTypes).avgLineCount shouldBe 7.5 } it("can find longest file") { diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index 92649ed..8ec4c02 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -5,11 +5,12 @@ import tutor.utils.FileUtil class DirectoryScannerSpec extends FunSpec with ShouldMatchers { describe("DirectoryScanner"){ - it("can scan directory recursively and return all file paths"){ + it("can scan directory recursively and return all file paths" + + " and it should only accept known txt files"){ val ds = new DirectoryScanner {} - val files = ds.scan("src/test/resources") - files.length shouldBe 3 - FileUtil.extractLocalPath(files.head) shouldBe "sourceFileSample" + val files = ds.scan("src/test/resources", Set("scala","java")) + files.length shouldBe 2 + FileUtil.extractLocalPath(files.head) shouldBe "SomeCode.scala" } } } From d19e423fff838fbde6c12463ad4c0a05957926dd Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 16:56:03 +0800 Subject: [PATCH 026/116] cover the case where the given directory is illegal --- src/main/scala/tutor/CodebaseAnalyzer.scala | 14 ++++++---- src/main/scala/tutor/DirectoryScanner.scala | 26 +++++++++++-------- src/main/scala/tutor/MainApp.scala | 2 +- .../scala/tutor/CodebaseAnalyzerSpec.scala | 26 ++++++++++++++----- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 039ce95..6c31042 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -11,11 +11,15 @@ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avg trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => - def analyze(path: Path, knownFileTypes: Set[String]): CodebaseInfo = { - val files = scan(path,knownFileTypes) - val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile) - val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length - CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos),avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos)) + def analyze(path: Path, knownFileTypes: Set[String]): Option[CodebaseInfo] = { + val files = scan(path, knownFileTypes) + if (files.isEmpty) { + None + } else { + val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile) + val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) + } } private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 2928588..c3daa35 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -16,18 +16,22 @@ trait DirectoryScanner extends StrictLogging { */ def scan(path: Path, knownFileTypes: Set[String]): Seq[Path] = { logger.info(s"scanning $path for known file types $knownFileTypes") - val file = new File(path) - val files = file.listFiles() - files.foldLeft(Vector[Path]()) { (acc, f) => - if (f.isFile) { - if(shouldAccept(f.getPath, knownFileTypes)) { - logger.info(s"add file to fold ${f.getAbsolutePath}") - acc :+ f.getAbsolutePath - }else{ - acc + val files = new File(path).listFiles() + if (files == null) { + logger.warn(s"$path is not a legal directory") + Vector[Path]() + } else { + files.foldLeft(Vector[Path]()) { (acc, f) => + if (f.isFile) { + if (shouldAccept(f.getPath, knownFileTypes)) { + logger.info(s"add file to fold ${f.getAbsolutePath}") + acc :+ f.getAbsolutePath + } else { + acc + } + } else { + acc ++ scan(f.getAbsolutePath, knownFileTypes) } - } else { - acc ++ scan(f.getAbsolutePath, knownFileTypes) } } } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index ca55bb0..1af0f90 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,7 +15,7 @@ object MainApp extends App with ReportFormatter with WriteSupport { val rs = if (file.isFile) { format(analyzer.processFile(file.getAbsolutePath)) } else { - format(analyzer.analyze(path, KnowFileTypes.knownFileTypes)) + analyzer.analyze(path, KnowFileTypes.knownFileTypes).map(format).getOrElse("not result found") } if (args.length > 1) { val output = args(1).drop(2) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index a3b07c3..e197524 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -7,7 +7,7 @@ import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { - override def scan(path: Path,knowFileTypes: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") + override def scan(path: Path, knowFileTypes: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): SourceCodeInfo = path match { case "a.scala" => SourceCodeInfo(path, path, 10) @@ -23,23 +23,35 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { describe("CodebaseAnalyzer") { it("can count file numbers by type") { val ls = List("a.scala", "b.scala", "c.sbt", "d") - codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String,Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } it("can analyze avg file count") { - codeBaseAnalyzer.analyze("anypath",KnowFileTypes.knownFileTypes).avgLineCount shouldBe 7.5 + codeBaseAnalyzer.analyze("anypath", KnowFileTypes.knownFileTypes).get.avgLineCount shouldBe 7.5 } it("can find longest file") { - codeBaseAnalyzer.longestFile(List(aInfo, bInfo)) shouldBe aInfo } - it("will return top 10 longest files"){ + it("will return top 10 longest files") { val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) val top10LongFiles = codeBaseAnalyzer.top10Files(sourceCodeInfos) top10LongFiles should have size 10 - top10LongFiles should not contain SourceCodeInfo("1.scala","1.scala",1) + top10LongFiles should not contain SourceCodeInfo("1.scala", "1.scala", 1) } - it("can count total line numbers"){ + it("can count total line numbers") { codeBaseAnalyzer.totalLineCount(List(aInfo, bInfo)) shouldBe 15 } + it("when directory scanner returns empty, code analyzer should return None") { + val emptyCodeAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { + override def scan(path: Path, knowFileTypes: Set[String]): Seq[Path] = Vector[Path]() + + override def processFile(path: Path): SourceCodeInfo = path match { + case "a.scala" => SourceCodeInfo(path, path, 10) + case "b.scala" => SourceCodeInfo(path, path, 10) + case "c.sbt" => SourceCodeInfo(path, path, 5) + case "d" => SourceCodeInfo(path, path, 5) + } + } + emptyCodeAnalyzer.analyze("anypath", KnowFileTypes.knownFileTypes) shouldBe None + } } } From 61c42c1d071b72fce0d44f828f1c4698da128284 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:09:26 +0800 Subject: [PATCH 027/116] add output to show how many times elapsed for analyze --- src/main/scala/tutor/MainApp.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 1af0f90..21e948e 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,11 +1,14 @@ package tutor import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil.Path import tutor.utils.WriteSupport -object MainApp extends App with ReportFormatter with WriteSupport { +object MainApp extends App with ReportFormatter with WriteSupport with StrictLogging { if (args.length < 1) { println("usage: CodeAnalyzer FilePath [-oOutputfile]") } else { @@ -15,7 +18,15 @@ object MainApp extends App with ReportFormatter with WriteSupport { val rs = if (file.isFile) { format(analyzer.processFile(file.getAbsolutePath)) } else { - analyzer.analyze(path, KnowFileTypes.knownFileTypes).map(format).getOrElse("not result found") + logger.info("start analyzing...") + val beginTime = new Date + val anayRs = analyzer.analyze(path, KnowFileTypes.knownFileTypes).map(format).getOrElse("not result found") + logger.info("analyze complete") + val endTime = new Date + val elapsed = new Date(endTime.getTime - beginTime.getTime) + val sdf = new SimpleDateFormat("hh:mm:ss.SSS") + logger.info(s"total elapsed ${sdf.format(elapsed)}") + anayRs } if (args.length > 1) { val output = args(1).drop(2) From 24b6f2fe0175d5cd26afb3b137a2747e23e37dc2 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:37:40 +0800 Subject: [PATCH 028/116] add ignore folders to ignore output files, eg, target in scala and java, bin in other languages --- src/main/scala/tutor/CodebaseAnalyzer.scala | 4 ++-- src/main/scala/tutor/DirectoryScanner.scala | 18 +++++++++--------- src/main/scala/tutor/MainApp.scala | 3 ++- ...KnowFileTypes.scala => PresetFilters.scala} | 4 +++- .../scala/tutor/CodebaseAnalyzerSpec.scala | 8 ++++---- .../scala/tutor/DirectoryScannerSpec.scala | 2 +- 6 files changed, 21 insertions(+), 18 deletions(-) rename src/main/scala/tutor/{KnowFileTypes.scala => PresetFilters.scala} (58%) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 6c31042..382648f 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -11,8 +11,8 @@ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avg trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => - def analyze(path: Path, knownFileTypes: Set[String]): Option[CodebaseInfo] = { - val files = scan(path, knownFileTypes) + def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { + val files = scan(path, knownFileTypes,ignoreFolders) if (files.isEmpty) { None } else { diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index c3daa35..b34c294 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -6,15 +6,18 @@ import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil import tutor.utils.FileUtil.Path +import scala.annotation.tailrec + trait DirectoryScanner extends StrictLogging { /** * recursively scan given directory, get all file path whose ext is in knownFileTypes Set * * @param path * @param knownFileTypes

file ext, like scala, java etc.

+ * @param ignoreFolders

used to ignore output folders, like target folder for scala and java; bin fold for other languages

* @return */ - def scan(path: Path, knownFileTypes: Set[String]): Seq[Path] = { + def scan(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = { logger.info(s"scanning $path for known file types $knownFileTypes") val files = new File(path).listFiles() if (files == null) { @@ -22,15 +25,12 @@ trait DirectoryScanner extends StrictLogging { Vector[Path]() } else { files.foldLeft(Vector[Path]()) { (acc, f) => - if (f.isFile) { - if (shouldAccept(f.getPath, knownFileTypes)) { - logger.info(s"add file to fold ${f.getAbsolutePath}") - acc :+ f.getAbsolutePath - } else { - acc - } + if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { + acc :+ f.getAbsolutePath + } else if (f.isDirectory && (!ignoreFolders.contains(FileUtil.extractLocalPath(f.getPath)))) { + acc ++ scan(f.getAbsolutePath, knownFileTypes, ignoreFolders) } else { - acc ++ scan(f.getAbsolutePath, knownFileTypes) + acc } } } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 21e948e..3276631 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -5,6 +5,7 @@ import java.text.SimpleDateFormat import java.util.Date import com.typesafe.scalalogging.slf4j.StrictLogging +import tutor.PresetFilters.{ignoreFolders, knownFileTypes} import tutor.utils.FileUtil.Path import tutor.utils.WriteSupport @@ -20,7 +21,7 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog } else { logger.info("start analyzing...") val beginTime = new Date - val anayRs = analyzer.analyze(path, KnowFileTypes.knownFileTypes).map(format).getOrElse("not result found") + val anayRs = analyzer.analyze(path, knownFileTypes, ignoreFolders).map(format).getOrElse("not result found") logger.info("analyze complete") val endTime = new Date val elapsed = new Date(endTime.getTime - beginTime.getTime) diff --git a/src/main/scala/tutor/KnowFileTypes.scala b/src/main/scala/tutor/PresetFilters.scala similarity index 58% rename from src/main/scala/tutor/KnowFileTypes.scala rename to src/main/scala/tutor/PresetFilters.scala index ede18e2..b8b4fa8 100644 --- a/src/main/scala/tutor/KnowFileTypes.scala +++ b/src/main/scala/tutor/PresetFilters.scala @@ -1,6 +1,8 @@ package tutor -object KnowFileTypes { +object PresetFilters { val knownFileTypes: Set[String] = Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp") + val ignoreFolders: Set[String] = + Set("target","bin") } diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index e197524..64f6e61 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -7,7 +7,7 @@ import tutor.utils.FileUtil.Path class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { - override def scan(path: Path, knowFileTypes: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") + override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): SourceCodeInfo = path match { case "a.scala" => SourceCodeInfo(path, path, 10) @@ -26,7 +26,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } it("can analyze avg file count") { - codeBaseAnalyzer.analyze("anypath", KnowFileTypes.knownFileTypes).get.avgLineCount shouldBe 7.5 + codeBaseAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders).get.avgLineCount shouldBe 7.5 } it("can find longest file") { codeBaseAnalyzer.longestFile(List(aInfo, bInfo)) shouldBe aInfo @@ -42,7 +42,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { } it("when directory scanner returns empty, code analyzer should return None") { val emptyCodeAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { - override def scan(path: Path, knowFileTypes: Set[String]): Seq[Path] = Vector[Path]() + override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): SourceCodeInfo = path match { case "a.scala" => SourceCodeInfo(path, path, 10) @@ -51,7 +51,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { case "d" => SourceCodeInfo(path, path, 5) } } - emptyCodeAnalyzer.analyze("anypath", KnowFileTypes.knownFileTypes) shouldBe None + emptyCodeAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) shouldBe None } } } diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index 8ec4c02..c1ead27 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -8,7 +8,7 @@ class DirectoryScannerSpec extends FunSpec with ShouldMatchers { it("can scan directory recursively and return all file paths" + " and it should only accept known txt files"){ val ds = new DirectoryScanner {} - val files = ds.scan("src/test/resources", Set("scala","java")) + val files = ds.scan("src/test/fixture", Set("scala","java"), Set("target")) files.length shouldBe 2 FileUtil.extractLocalPath(files.head) shouldBe "SomeCode.scala" } From 05a83c726c4209da033c6058dbde29c0cdf2d660 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:38:06 +0800 Subject: [PATCH 029/116] move test data to fixture folder instead of resources folder --- src/test/{resources => fixture}/sourceFileSample | 0 src/test/{resources => fixture}/sub/SomeCode.scala | 0 src/test/{resources => fixture}/sub/sub1/ignore.me | 0 src/test/{resources => fixture}/sub/sub1/othercode.java | 0 src/test/scala/tutor/SourceCodeAnalyzerSpec.scala | 4 ++-- 5 files changed, 2 insertions(+), 2 deletions(-) rename src/test/{resources => fixture}/sourceFileSample (100%) rename src/test/{resources => fixture}/sub/SomeCode.scala (100%) rename src/test/{resources => fixture}/sub/sub1/ignore.me (100%) rename src/test/{resources => fixture}/sub/sub1/othercode.java (100%) diff --git a/src/test/resources/sourceFileSample b/src/test/fixture/sourceFileSample similarity index 100% rename from src/test/resources/sourceFileSample rename to src/test/fixture/sourceFileSample diff --git a/src/test/resources/sub/SomeCode.scala b/src/test/fixture/sub/SomeCode.scala similarity index 100% rename from src/test/resources/sub/SomeCode.scala rename to src/test/fixture/sub/SomeCode.scala diff --git a/src/test/resources/sub/sub1/ignore.me b/src/test/fixture/sub/sub1/ignore.me similarity index 100% rename from src/test/resources/sub/sub1/ignore.me rename to src/test/fixture/sub/sub1/ignore.me diff --git a/src/test/resources/sub/sub1/othercode.java b/src/test/fixture/sub/sub1/othercode.java similarity index 100% rename from src/test/resources/sub/sub1/othercode.java rename to src/test/fixture/sub/sub1/othercode.java diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 79dd901..9e8be7d 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -6,9 +6,9 @@ class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers{ describe("SourceCode object"){ it("can read file and create a SourceCode instance"){ val sca = new SourceCodeAnalyzer {} - val sourceCodeInfo = sca.processFile("./src/test/resources/sourceFileSample") + val sourceCodeInfo = sca.processFile("./src/test/fixture/sourceFileSample") sourceCodeInfo.localPath shouldBe "sourceFileSample" - sourceCodeInfo.path shouldBe "./src/test/resources/sourceFileSample" + sourceCodeInfo.path shouldBe "./src/test/fixture/sourceFileSample" sourceCodeInfo.count shouldBe 108 } } From 356a183ec4209b2ed1138525f3442310677ea02e Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:39:18 +0800 Subject: [PATCH 030/116] optimize imports --- src/main/scala/tutor/DirectoryScanner.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index b34c294..cd24bc8 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -6,8 +6,6 @@ import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil import tutor.utils.FileUtil.Path -import scala.annotation.tailrec - trait DirectoryScanner extends StrictLogging { /** * recursively scan given directory, get all file path whose ext is in knownFileTypes Set From 35bde1892db2adc3a492e97c4cb00abcbe9308c6 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:52:06 +0800 Subject: [PATCH 031/116] report should show absolute path --- src/main/scala/tutor/ReportFormatter.scala | 4 +-- .../scala/tutor/ReportFormatterSpec.scala | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index 72b9a79..ce312fc 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -10,12 +10,12 @@ trait ReportFormatter { ReportFormatter.separator ++ "\n\n" ++ s"total line count: ${codebaseInfo.totalLineCount}" ++ "\n" ++ s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ - s"longest file: ${longestFileInfo.localPath} ${longestFileInfo.count}" ++ + s"longest file: ${longestFileInfo.path} ${longestFileInfo.count}" ++ "\n" ++ ReportFormatter.separator ++ "\n\n" ++ "top 10 long files\n" ++ codebaseInfo.top10Files.map { - s => s"${s.localPath} ${s.count}" + s => s"${s.path} ${s.count}" }.mkString("\n") } diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 9b85cee..0ba6b51 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -15,8 +15,8 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), totalLineCount = 15, avgLineCount = 7.5, - SourceCodeInfo("a.scala", "a.scala", 10), { - for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) + SourceCodeInfo("absolute/a.scala", "a.scala", 10), { + for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"absolute/$i.scala", s"$i.scala", i) } ) rf.format(codebaseInfo) shouldBe @@ -28,20 +28,20 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { | |total line count: 15 |avg line count: 7.5 - |longest file: a.scala 10 + |longest file: absolute/a.scala 10 |${ReportFormatter.separator} | |top 10 long files - |10.scala 10 - |9.scala 9 - |8.scala 8 - |7.scala 7 - |6.scala 6 - |5.scala 5 - |4.scala 4 - |3.scala 3 - |2.scala 2 - |1.scala 1 + |absolute/10.scala 10 + |absolute/9.scala 9 + |absolute/8.scala 8 + |absolute/7.scala 7 + |absolute/6.scala 6 + |absolute/5.scala 5 + |absolute/4.scala 4 + |absolute/3.scala 3 + |absolute/2.scala 2 + |absolute/1.scala 1 """.trim.stripMargin } } From 218df32c1e478d4ecb796fd34ce17fe188f94874 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 17:56:17 +0800 Subject: [PATCH 032/116] minutes should be enough --- src/main/scala/tutor/MainApp.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 3276631..0ad3de5 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -25,7 +25,7 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog logger.info("analyze complete") val endTime = new Date val elapsed = new Date(endTime.getTime - beginTime.getTime) - val sdf = new SimpleDateFormat("hh:mm:ss.SSS") + val sdf = new SimpleDateFormat("mm:ss.SSS") logger.info(s"total elapsed ${sdf.format(elapsed)}") anayRs } From 7b5420c9720102293ba99bf1b05c81684575e986 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:22:51 +0800 Subject: [PATCH 033/116] Try and filter out those files that are not utf-8 encoded --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- src/main/scala/tutor/MainApp.scala | 2 +- src/main/scala/tutor/SourceCodeInfo.scala | 20 ++++++++++++------- .../scala/tutor/CodebaseAnalyzerSpec.scala | 19 ++++++++---------- .../scala/tutor/SourceCodeAnalyzerSpec.scala | 2 +- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 382648f..81dc58e 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -16,7 +16,7 @@ trait CodebaseAnalyzer { if (files.isEmpty) { None } else { - val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile) + val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile).filter(_.isSuccess).map(_.get) val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 0ad3de5..0616310 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -17,7 +17,7 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val file = new File(path) val analyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer val rs = if (file.isFile) { - format(analyzer.processFile(file.getAbsolutePath)) + analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") } else { logger.info("start analyzing...") val beginTime = new Date diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index c94c047..276e2e4 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -4,20 +4,26 @@ import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.FileUtil._ import tutor.utils.FileUtil +import scala.util.Try + case class SourceCodeInfo(path: String, localPath: String, count: Int) -object SourceCodeInfo{ + +object SourceCodeInfo { + implicit object SourceCodeInfoOrdering extends Ordering[SourceCodeInfo] { override def compare(x: SourceCodeInfo, y: SourceCodeInfo): Int = x.count compare y.count } + } -trait SourceCodeAnalyzer extends StrictLogging{ - def processFile(path: Path): SourceCodeInfo = { +trait SourceCodeAnalyzer extends StrictLogging { + def processFile(path: Path): Try[SourceCodeInfo] = { import scala.io._ logger.info(s"processing $path") - val source = Source.fromFile(path) - val lines = source.getLines.toList - SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) + Try { + val source = Source.fromFile(path) + val lines = source.getLines.toList + SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) + } } - } \ No newline at end of file diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 64f6e61..db97ad8 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -4,16 +4,18 @@ import org.scalatest.{FunSpec, ShouldMatchers} import tutor.utils.FileUtil import tutor.utils.FileUtil.Path +import scala.util.{Success, Try} + class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") - override def processFile(path: Path): SourceCodeInfo = path match { - case "a.scala" => SourceCodeInfo(path, path, 10) - case "b.scala" => SourceCodeInfo(path, path, 10) - case "c.sbt" => SourceCodeInfo(path, path, 5) - case "d" => SourceCodeInfo(path, path, 5) + override def processFile(path: Path): Try[SourceCodeInfo] = path match { + case "a.scala" => Success(SourceCodeInfo(path, path, 10)) + case "b.scala" => Success(SourceCodeInfo(path, path, 10)) + case "c.sbt" => Success(SourceCodeInfo(path, path, 5)) + case "d" => Success(SourceCodeInfo(path, path, 5)) } } @@ -44,12 +46,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val emptyCodeAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() - override def processFile(path: Path): SourceCodeInfo = path match { - case "a.scala" => SourceCodeInfo(path, path, 10) - case "b.scala" => SourceCodeInfo(path, path, 10) - case "c.sbt" => SourceCodeInfo(path, path, 5) - case "d" => SourceCodeInfo(path, path, 5) - } + override def processFile(path: Path): Try[SourceCodeInfo] = ??? } emptyCodeAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) shouldBe None } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 9e8be7d..6e3d3c1 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -6,7 +6,7 @@ class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers{ describe("SourceCode object"){ it("can read file and create a SourceCode instance"){ val sca = new SourceCodeAnalyzer {} - val sourceCodeInfo = sca.processFile("./src/test/fixture/sourceFileSample") + val sourceCodeInfo = sca.processFile("./src/test/fixture/sourceFileSample").get sourceCodeInfo.localPath shouldBe "sourceFileSample" sourceCodeInfo.path shouldBe "./src/test/fixture/sourceFileSample" sourceCodeInfo.count shouldBe 108 From b57bdb91ed9e0f7f789837fa97589ca669032e70 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:33:17 +0800 Subject: [PATCH 034/116] remove log outputs --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- src/main/scala/tutor/DirectoryScanner.scala | 2 -- src/main/scala/tutor/SourceCodeInfo.scala | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 81dc58e..83d5528 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -16,7 +16,7 @@ trait CodebaseAnalyzer { if (files.isEmpty) { None } else { - val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile).filter(_.isSuccess).map(_.get) + val sourceCodeInfos: Seq[SourceCodeInfo] = files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index cd24bc8..58368da 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -16,7 +16,6 @@ trait DirectoryScanner extends StrictLogging { * @return */ def scan(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = { - logger.info(s"scanning $path for known file types $knownFileTypes") val files = new File(path).listFiles() if (files == null) { logger.warn(s"$path is not a legal directory") @@ -35,7 +34,6 @@ trait DirectoryScanner extends StrictLogging { } private def shouldAccept(path: Path, knownFileTypes: Set[String]): Boolean = { - logger.info(s"check if should accept $path") knownFileTypes.contains(FileUtil.extractExtFileName(path)) } } diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index 276e2e4..bdb275c 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -19,7 +19,6 @@ object SourceCodeInfo { trait SourceCodeAnalyzer extends StrictLogging { def processFile(path: Path): Try[SourceCodeInfo] = { import scala.io._ - logger.info(s"processing $path") Try { val source = Source.fromFile(path) val lines = source.getLines.toList From 4cddfc7441ddff7d290a4d5076c9c3c4e5ee25ba Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:44:43 +0800 Subject: [PATCH 035/116] return to sequence implementation --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 83d5528..81dc58e 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -16,7 +16,7 @@ trait CodebaseAnalyzer { if (files.isEmpty) { None } else { - val sourceCodeInfos: Seq[SourceCodeInfo] = files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector + val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile).filter(_.isSuccess).map(_.get) val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } From f62c354d4f27058e780120a95091b8a64431313c Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:45:04 +0800 Subject: [PATCH 036/116] add more file types to scan --- src/main/scala/tutor/PresetFilters.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/PresetFilters.scala b/src/main/scala/tutor/PresetFilters.scala index b8b4fa8..a7deb99 100644 --- a/src/main/scala/tutor/PresetFilters.scala +++ b/src/main/scala/tutor/PresetFilters.scala @@ -2,7 +2,7 @@ package tutor object PresetFilters { val knownFileTypes: Set[String] = - Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp") + Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp", "hs", "properties","sbt","js","html") val ignoreFolders: Set[String] = Set("target","bin") } From daec35a8ee59f288608833f96dce1af4ff339a7b Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:46:47 +0800 Subject: [PATCH 037/116] ignore .idea folder --- src/main/scala/tutor/PresetFilters.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/PresetFilters.scala b/src/main/scala/tutor/PresetFilters.scala index a7deb99..57848d0 100644 --- a/src/main/scala/tutor/PresetFilters.scala +++ b/src/main/scala/tutor/PresetFilters.scala @@ -4,5 +4,5 @@ object PresetFilters { val knownFileTypes: Set[String] = Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp", "hs", "properties","sbt","js","html") val ignoreFolders: Set[String] = - Set("target","bin") + Set("target","bin",".idea") } From bf5405650312551f0abb10616d07c01a14eda900 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:52:24 +0800 Subject: [PATCH 038/116] print scanned directory name so that we can see progress when run on large code base --- src/main/scala/tutor/DirectoryScanner.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 58368da..59b8e3a 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -22,10 +22,12 @@ trait DirectoryScanner extends StrictLogging { Vector[Path]() } else { files.foldLeft(Vector[Path]()) { (acc, f) => + val filePath = f.getAbsolutePath if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { - acc :+ f.getAbsolutePath + acc :+ filePath } else if (f.isDirectory && (!ignoreFolders.contains(FileUtil.extractLocalPath(f.getPath)))) { - acc ++ scan(f.getAbsolutePath, knownFileTypes, ignoreFolders) + logger.info(s"directory $filePath scanned and added") + acc ++ scan(filePath, knownFileTypes, ignoreFolders) } else { acc } From b8196e3156723a0b49f6e231f3796f0470a19d59 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 18:53:59 +0800 Subject: [PATCH 039/116] ignore .git folder --- src/main/scala/tutor/PresetFilters.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/PresetFilters.scala b/src/main/scala/tutor/PresetFilters.scala index 57848d0..3d3c0f1 100644 --- a/src/main/scala/tutor/PresetFilters.scala +++ b/src/main/scala/tutor/PresetFilters.scala @@ -4,5 +4,5 @@ object PresetFilters { val knownFileTypes: Set[String] = Set("scala", "java", "txt", "xml", "json", "c", "h", "cpp", "hs", "properties","sbt","js","html") val ignoreFolders: Set[String] = - Set("target","bin",".idea") + Set("target","bin",".idea",".git") } From da7c5c8dfb88c74ad5d3e213573181b2f96f7f8b Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 22:13:49 +0800 Subject: [PATCH 040/116] create util to make benchmark easier --- .../scala/tutor/utils/BenchmarkUtil.scala | 20 +++++++++++++++++++ .../scala/tutor/utils/BenchmarkUtilSpec.scala | 14 +++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/main/scala/tutor/utils/BenchmarkUtil.scala create mode 100644 src/test/scala/tutor/utils/BenchmarkUtilSpec.scala diff --git a/src/main/scala/tutor/utils/BenchmarkUtil.scala b/src/main/scala/tutor/utils/BenchmarkUtil.scala new file mode 100644 index 0000000..f811baa --- /dev/null +++ b/src/main/scala/tutor/utils/BenchmarkUtil.scala @@ -0,0 +1,20 @@ +package tutor.utils + +import java.text.SimpleDateFormat +import java.util.Date + +import com.typesafe.scalalogging.slf4j.StrictLogging + +object BenchmarkUtil extends StrictLogging { + def record[T](actionDesc: String)(action: => T): T = { + val beginTime = new Date + logger.info(s"begin $actionDesc") + val rs = action + logger.info(s"end $actionDesc") + val endTime = new Date + val elapsed = new Date(endTime.getTime - beginTime.getTime) + val sdf = new SimpleDateFormat("mm:ss.SSS") + logger.info(s"$actionDesc total elapsed ${sdf.format(elapsed)}") + rs + } +} diff --git a/src/test/scala/tutor/utils/BenchmarkUtilSpec.scala b/src/test/scala/tutor/utils/BenchmarkUtilSpec.scala new file mode 100644 index 0000000..cf68163 --- /dev/null +++ b/src/test/scala/tutor/utils/BenchmarkUtilSpec.scala @@ -0,0 +1,14 @@ +package tutor.utils + +import org.scalatest.FunSpec + +class BenchmarkUtilSpec extends FunSpec { + describe("BenchmarkUtil") { + it("records the start time and end time of an action, and calculate the elapsed time") { + //this test is not easy to verify, so just run and look the result + BenchmarkUtil.record("sleep") { + Thread.sleep(100) + } + } + } +} From be28bbed21099a4391f259221411d25720fa9443 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 22:34:15 +0800 Subject: [PATCH 041/116] should close file after reading --- src/main/scala/tutor/SourceCodeInfo.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index bdb275c..f56c6c9 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -21,8 +21,12 @@ trait SourceCodeAnalyzer extends StrictLogging { import scala.io._ Try { val source = Source.fromFile(path) - val lines = source.getLines.toList - SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) + try { + val lines = source.getLines.toList + SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) + }finally { + source.close() + } } } } \ No newline at end of file From 6f0a145bbc339dbf730fbacf131070784210644b Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 22:34:26 +0800 Subject: [PATCH 042/116] add benchmark --- src/main/scala/tutor/CodebaseAnalyzer.scala | 16 +++++++++++----- src/main/scala/tutor/MainApp.scala | 14 ++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 81dc58e..88983b7 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -1,6 +1,6 @@ package tutor -import tutor.utils.FileUtil +import tutor.utils.{BenchmarkUtil, FileUtil} import tutor.utils.FileUtil._ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avgLineCount: Double, @@ -12,13 +12,19 @@ trait CodebaseAnalyzer { this: DirectoryScanner with SourceCodeAnalyzer => def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { - val files = scan(path, knownFileTypes,ignoreFolders) + val files = BenchmarkUtil.record("scan folders") { + scan(path, knownFileTypes, ignoreFolders) + } if (files.isEmpty) { None } else { - val sourceCodeInfos: Seq[SourceCodeInfo] = files.map(processFile).filter(_.isSuccess).map(_.get) - val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length - Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) + val sourceCodeInfos: Seq[SourceCodeInfo] = BenchmarkUtil.record("processing each file") { + files.map(processFile).filter(_.isSuccess).map(_.get) + } + BenchmarkUtil.record("make last result") { + val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) + } } } diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 0616310..7ab49fc 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -7,7 +7,7 @@ import java.util.Date import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.PresetFilters.{ignoreFolders, knownFileTypes} import tutor.utils.FileUtil.Path -import tutor.utils.WriteSupport +import tutor.utils.{BenchmarkUtil, WriteSupport} object MainApp extends App with ReportFormatter with WriteSupport with StrictLogging { if (args.length < 1) { @@ -19,15 +19,9 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") } else { - logger.info("start analyzing...") - val beginTime = new Date - val anayRs = analyzer.analyze(path, knownFileTypes, ignoreFolders).map(format).getOrElse("not result found") - logger.info("analyze complete") - val endTime = new Date - val elapsed = new Date(endTime.getTime - beginTime.getTime) - val sdf = new SimpleDateFormat("mm:ss.SSS") - logger.info(s"total elapsed ${sdf.format(elapsed)}") - anayRs + BenchmarkUtil.record(s"analyze code under $path") { + analyzer.analyze(path, knownFileTypes, ignoreFolders).map(format).getOrElse("not result found") + } } if (args.length > 1) { val output = args(1).drop(2) From 31f104ab2698565682336fbe6a94c10a71a51abc Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 23:13:59 +0800 Subject: [PATCH 043/116] seperate seq impl and par impl --- src/main/scala/tutor/CodebaseAnalyzer.scala | 5 ++++- src/main/scala/tutor/CodebaseAnalyzerParImpl.scala | 11 +++++++++++ src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala | 11 +++++++++++ src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 4 ++-- 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 src/main/scala/tutor/CodebaseAnalyzerParImpl.scala create mode 100644 src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 88983b7..c7412af 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -19,7 +19,7 @@ trait CodebaseAnalyzer { None } else { val sourceCodeInfos: Seq[SourceCodeInfo] = BenchmarkUtil.record("processing each file") { - files.map(processFile).filter(_.isSuccess).map(_.get) + processSourceFiles(files) } BenchmarkUtil.record("make last result") { val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length @@ -28,6 +28,9 @@ trait CodebaseAnalyzer { } } + + protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] + private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) } diff --git a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala new file mode 100644 index 0000000..ff24d64 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala @@ -0,0 +1,11 @@ +package tutor + +import tutor.utils.FileUtil.Path + +trait CodebaseAnalyzerParImpl extends CodebaseAnalyzer { + this: DirectoryScanner with SourceCodeAnalyzer => + + override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { + files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector + } +} diff --git a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala new file mode 100644 index 0000000..8c6dce1 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala @@ -0,0 +1,11 @@ +package tutor + +import tutor.utils.FileUtil.Path + +trait CodebaseAnalyzerSeqImpl extends CodebaseAnalyzer { + this: DirectoryScanner with SourceCodeAnalyzer => + + override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { + files.map(processFile).filter(_.isSuccess).map(_.get) + } +} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index db97ad8..8fbb0ba 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -8,7 +8,7 @@ import scala.util.{Success, Try} class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { - val codeBaseAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { + val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): Try[SourceCodeInfo] = path match { @@ -43,7 +43,7 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { codeBaseAnalyzer.totalLineCount(List(aInfo, bInfo)) shouldBe 15 } it("when directory scanner returns empty, code analyzer should return None") { - val emptyCodeAnalyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer { + val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? From 27feed1f7c99782998950a91395367430de3b382 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 23:14:56 +0800 Subject: [PATCH 044/116] add -p to use par mode --- src/main/scala/tutor/MainApp.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 7ab49fc..13bbfa7 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,7 +15,13 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog } else { val path: Path = args(0) val file = new File(path) - val analyzer = new CodebaseAnalyzer with DirectoryScanner with SourceCodeAnalyzer + val analyzer = args.find(_.startsWith("-p")).map { _ => + logger.info("using par collection mode") + new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer + }.getOrElse { + logger.info("using sequence collection mode") + new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer + } val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") } else { @@ -23,15 +29,14 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog analyzer.analyze(path, knownFileTypes, ignoreFolders).map(format).getOrElse("not result found") } } - if (args.length > 1) { - val output = args(1).drop(2) + args.find(_.startsWith("-o")).foreach { opt => + val output = opt.drop(2) withWriter(output) { _.write(rs) } println(s"report saved into $output") - } else { - println(rs) } + println(rs) } } From 3ca342165cfcecc4384d02e6fd0ff8d9391bb9eb Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 23:15:32 +0800 Subject: [PATCH 045/116] output too many logs --- src/main/scala/tutor/DirectoryScanner.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 59b8e3a..27d836d 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -26,7 +26,6 @@ trait DirectoryScanner extends StrictLogging { if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { acc :+ filePath } else if (f.isDirectory && (!ignoreFolders.contains(FileUtil.extractLocalPath(f.getPath)))) { - logger.info(s"directory $filePath scanned and added") acc ++ scan(filePath, knownFileTypes, ignoreFolders) } else { acc From 58d0980bc551c3f8fe19c59d5de8107d1f36fbf0 Mon Sep 17 00:00:00 2001 From: notyy Date: Sat, 21 Jan 2017 23:26:34 +0800 Subject: [PATCH 046/116] add akka dependency --- build.sbt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7d7eaf1..36e1b4e 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,18 @@ libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "2.2.1" % "test", "org.slf4j" % "slf4j-api" % "1.7.7", "ch.qos.logback" % "logback-classic" % "1.1.2", - "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2" + "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", + "com.typesafe.akka" %% "akka-actor" % "2.4.7", + "com.typesafe.akka" %% "akka-http-core" % "2.4.7", + "com.typesafe.akka" %% "akka-http-testkit" % "2.4.7", + "com.typesafe.akka" %% "akka-persistence" % "2.4.7", + "com.typesafe.akka" %% "akka-persistence-tck" % "2.4.7", + "com.typesafe.akka" %% "akka-slf4j" % "2.4.7", + "com.typesafe.akka" %% "akka-stream" % "2.4.7", + "com.typesafe.akka" %% "akka-stream-testkit" % "2.4.7", + "com.typesafe.akka" %% "akka-testkit" % "2.4.7", + "com.typesafe.akka" %% "akka-http-experimental" % "2.4.7", + "com.typesafe.akka" %% "akka-http-jackson-experimental" % "2.4.7" ) // TODO reopen it later From 9bf55098b62355df268852a5f593528b4e36cc31 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 27 Feb 2017 20:51:21 +0800 Subject: [PATCH 047/116] trigger jenkins --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index c7412af..d1f4b60 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -21,7 +21,7 @@ trait CodebaseAnalyzer { val sourceCodeInfos: Seq[SourceCodeInfo] = BenchmarkUtil.record("processing each file") { processSourceFiles(files) } - BenchmarkUtil.record("make last result") { + BenchmarkUtil.record("make last result ##") { val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } From 98f907ea785771a0df2e0a6b471214a97cb06a5f Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 1 Mar 2017 14:22:49 +0800 Subject: [PATCH 048/116] remove useless imports --- src/main/scala/tutor/MainApp.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 13bbfa7..0248adf 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -1,8 +1,6 @@ package tutor import java.io.File -import java.text.SimpleDateFormat -import java.util.Date import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.PresetFilters.{ignoreFolders, knownFileTypes} From 33ce72f8cb8962b97812d94a4eeb4e1d5b1b655e Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 1 Mar 2017 23:59:21 +0800 Subject: [PATCH 049/116] try jenkins file --- Jenkinsfile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..5c35caf --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,10 @@ +pipeline { + agent any + stages { + stage('build') { + steps { + sh 'sbt clean compile' + } + } + } +} \ No newline at end of file From 3ef1a0cb745167de0a493ff2a20c50dd87289698 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 00:05:23 +0800 Subject: [PATCH 050/116] add an assembly stage in jenkins pipeline --- Jenkinsfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5c35caf..05c2a8b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,12 @@ pipeline { stages { stage('build') { steps { - sh 'sbt clean compile' + sh 'sbt clean compile test' + } + } + stage('assembly') { + steps { + sh 'sbt assembly' } } } From 7064c466cd5bf766e6f275b2bfb5fa8264914088 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 00:40:20 +0800 Subject: [PATCH 051/116] add tag for functional test --- src/test/scala/tags/TestTypeTag.scala | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/test/scala/tags/TestTypeTag.scala diff --git a/src/test/scala/tags/TestTypeTag.scala b/src/test/scala/tags/TestTypeTag.scala new file mode 100644 index 0000000..9c2055f --- /dev/null +++ b/src/test/scala/tags/TestTypeTag.scala @@ -0,0 +1,7 @@ +package tags + +import org.scalatest.Tag + +object TestTypeTag { + object FunctionalTest extends Tag("com.github.notyy.codeAnalyzer.FunctionalTest") +} From c6e848e50f018b4b253b2ddb2bdab912779a55d9 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 00:40:50 +0800 Subject: [PATCH 052/116] set functionalTest tag on some tests --- src/test/scala/tutor/DirectoryScannerSpec.scala | 7 ++++--- src/test/scala/tutor/SourceCodeAnalyzerSpec.scala | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index c1ead27..6ded74a 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -1,14 +1,15 @@ package tutor import org.scalatest.{FunSpec, ShouldMatchers} +import tags.TestTypeTag.FunctionalTest import tutor.utils.FileUtil class DirectoryScannerSpec extends FunSpec with ShouldMatchers { - describe("DirectoryScanner"){ + describe("DirectoryScanner") { it("can scan directory recursively and return all file paths" + - " and it should only accept known txt files"){ + " and it should only accept known txt files", FunctionalTest) { val ds = new DirectoryScanner {} - val files = ds.scan("src/test/fixture", Set("scala","java"), Set("target")) + val files = ds.scan("src/test/fixture", Set("scala", "java"), Set("target")) files.length shouldBe 2 FileUtil.extractLocalPath(files.head) shouldBe "SomeCode.scala" } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 6e3d3c1..749e60b 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -1,10 +1,11 @@ package tutor import org.scalatest.{FunSpec, ShouldMatchers} +import tags.TestTypeTag.FunctionalTest -class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers{ - describe("SourceCode object"){ - it("can read file and create a SourceCode instance"){ +class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers { + describe("SourceCode object") { + it("can read file and create a SourceCode instance", FunctionalTest) { val sca = new SourceCodeAnalyzer {} val sourceCodeInfo = sca.processFile("./src/test/fixture/sourceFileSample").get sourceCodeInfo.localPath shouldBe "sourceFileSample" From 1da704fc73695addd8920d7335846be8506dd6da Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 00:41:13 +0800 Subject: [PATCH 053/116] add functional test stage to jenkins pipeline --- Jenkinsfile | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 05c2a8b..07227e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,19 @@ pipeline { agent any stages { - stage('build') { + stage('compile') { steps { - sh 'sbt clean compile test' + sh 'sbt clean compile' + } + } + stage('unit test'{ + steps { + sh 'sbt "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + } + } + stage('functional test'{ + steps { + sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' } } stage('assembly') { From df41752e55625d0ec655d4161f528afeb0eb1216 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 00:42:38 +0800 Subject: [PATCH 054/116] fix type error --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07227e5..84194fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,12 +6,12 @@ pipeline { sh 'sbt clean compile' } } - stage('unit test'{ + stage('unit test') { steps { sh 'sbt "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' } } - stage('functional test'{ + stage('functional test') { steps { sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' } From 3a918a5dc30e7327a79237dd044acb33da45a665 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 10:59:50 +0800 Subject: [PATCH 055/116] ignore sonar files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c58d83b..c98832c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ project/plugins/project/ # Scala-IDE specific .scala_dependencies .worksheet +.sonar From 08ab3d4ebd43a46075fa276bba3e8b248d1083da Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:01:22 +0800 Subject: [PATCH 056/116] add scoverage plugin --- project/plugins.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 1741de9..f38daac 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1 +1,3 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") \ No newline at end of file +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") + +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") From ee4aed841082d1d913aabfe3a1d9dff211e72524 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:01:49 +0800 Subject: [PATCH 057/116] add test coverage stage in pipeline --- Jenkinsfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 84194fe..778ba6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,9 +6,14 @@ pipeline { sh 'sbt clean compile' } } - stage('unit test') { + stage('unit test with coverage') { steps { - sh 'sbt "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + sh 'sbt "coverage testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + } + } + stage('rebuild without coverage') { + steps { + sh 'sbt clean compile' } } stage('functional test') { From 7d0909f2c23efe6dd031c15f930eb5f958b2af6c Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:10:26 +0800 Subject: [PATCH 058/116] fix issue on code coverage --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 778ba6b..f544774 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,8 @@ pipeline { } stage('unit test with coverage') { steps { - sh 'sbt "coverage testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + sh 'coverageReport' } } stage('rebuild without coverage') { From 48141b2e70c6dd2652dcd00ebe8ee2a266256c2b Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:13:21 +0800 Subject: [PATCH 059/116] fix issue on code coverage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f544774..4f15d4b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,7 +9,7 @@ pipeline { stage('unit test with coverage') { steps { sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' - sh 'coverageReport' + sh 'sbt coverageReport' } } stage('rebuild without coverage') { From 2cbec3cba8c893597f355163b029324dbdf35193 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:30:45 +0800 Subject: [PATCH 060/116] mv scoverage report out of target folder --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 4f15d4b..c5bc7ae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,6 +10,8 @@ pipeline { steps { sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' sh 'sbt coverageReport' + sh 'mkdir report' + sh 'mv -r target/scala-2.11/scoverage-report ./report/' } } stage('rebuild without coverage') { From 9148eb5c7d275144b3ca2a28076aa09e7ce1f7af Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:34:51 +0800 Subject: [PATCH 061/116] mv scoverage report out of target folder --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c5bc7ae..ad08ce4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' sh 'sbt coverageReport' sh 'mkdir report' - sh 'mv -r target/scala-2.11/scoverage-report ./report/' + sh 'cp -R target/scala-2.11/scoverage-report ./report/' } } stage('rebuild without coverage') { From 1a32659dafefdd8bdd7e877ff16848a229a2c887 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 11:38:58 +0800 Subject: [PATCH 062/116] mv scoverage report out of target folder --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ad08ce4..18e8568 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,6 @@ pipeline { steps { sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' sh 'sbt coverageReport' - sh 'mkdir report' sh 'cp -R target/scala-2.11/scoverage-report ./report/' } } From cee00609df664eae4384a9ba9d2a59b946ea3b02 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 17:28:46 +0800 Subject: [PATCH 063/116] separate unit test and test coverage --- Jenkinsfile | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 18e8568..12876ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,9 +6,14 @@ pipeline { sh 'sbt clean compile' } } - stage('unit test with coverage') { + stage('unit test') { steps { - sh 'sbt coverage "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + sh 'sbt "testOnly * -- -l com.github.notyy.codeAnalyzer.FunctionalTest"' + } + } + stage('test coverage') { + steps { + sh 'sbt coverage test' sh 'sbt coverageReport' sh 'cp -R target/scala-2.11/scoverage-report ./report/' } @@ -18,14 +23,21 @@ pipeline { sh 'sbt clean compile' } } - stage('functional test') { - steps { - sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' + parallel { + stage('functional test') { + steps { + sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' + } + } + stage('assembly') { + steps { + sh 'sbt assembly' + } } } - stage('assembly') { + stage('deploy') { steps { - sh 'sbt assembly' + sh 'echo deploy' } } } From 0e9aa514c3d891b3e74e47f873e97a71cb61c99d Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 17:40:45 +0800 Subject: [PATCH 064/116] separate unit test and test coverage --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 12876ff..1a671b6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,7 +23,7 @@ pipeline { sh 'sbt clean compile' } } - parallel { + stage('FT&Assembly') { stage('functional test') { steps { sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' From c9d151cd4b4c09d1f189dac4bdd612d74001bdc0 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 18:18:46 +0800 Subject: [PATCH 065/116] jenkins declarative pipeline syntax doesn't support fork&join --- Jenkinsfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1a671b6..e7fb797 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,16 +23,14 @@ pipeline { sh 'sbt clean compile' } } - stage('FT&Assembly') { - stage('functional test') { - steps { - sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' - } + stage('functional test') { + steps { + sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' } - stage('assembly') { - steps { - sh 'sbt assembly' - } + } + stage('assembly') { + steps { + sh 'sbt assembly' } } stage('deploy') { From 24b4c75e4340461e613a4832ec110c9df1553d4e Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 2 Mar 2017 20:48:13 +0800 Subject: [PATCH 066/116] add predeploy test and health check stage placeholder --- Jenkinsfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e7fb797..cde0a3b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,10 +33,20 @@ pipeline { sh 'sbt assembly' } } + stage('predeploy test') { + steps { + sh 'echo predeploy test' + } + } stage('deploy') { steps { sh 'echo deploy' } } + stage('health check') { + steps { + sh 'echo health check' + } + } } } \ No newline at end of file From 29a1974f561170aac98598d0851f9a640c7d72f3 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 6 Mar 2017 00:41:08 +0800 Subject: [PATCH 067/116] add a shell file to easily run codeAnalyzer --- codeAnalyzer | 1 + 1 file changed, 1 insertion(+) create mode 100644 codeAnalyzer diff --git a/codeAnalyzer b/codeAnalyzer new file mode 100644 index 0000000..8358c43 --- /dev/null +++ b/codeAnalyzer @@ -0,0 +1 @@ +java -jar ~/dev/bin/CodeAnalyzer.jar $1 \ No newline at end of file From 18eaf5f1ca35e9e3335f89b11489fc3f4bc0b753 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 6 Mar 2017 00:42:55 +0800 Subject: [PATCH 068/116] change from FunSpec to FeatureSpec to demonstrate BDD --- .../scala/tutor/CodebaseAnalyzerSpec.scala | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 8fbb0ba..e3a36cc 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -1,12 +1,12 @@ package tutor -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FeatureSpec, FunSpec, GivenWhenThen, ShouldMatchers} import tutor.utils.FileUtil import tutor.utils.FileUtil.Path import scala.util.{Success, Try} -class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { +class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") @@ -22,32 +22,55 @@ class CodebaseAnalyzerSpec extends FunSpec with ShouldMatchers { val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) - describe("CodebaseAnalyzer") { - it("can count file numbers by type") { + info("As a technical consultant") + info("I want to analyze a customer's code base") + info("So that I can find bad smell in code base more easily") + + feature("Analyze a folder containing source code") { + scenario("count file numbers by type") { + Given("a folder contains 2 .scala file and a .sbt file and a file without ext") val ls = List("a.scala", "b.scala", "c.sbt", "d") + When("analyze that folder") + Then("result should contain 2 scala file, 1 sbt file and 1 empty-type file") codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) } - it("can analyze avg file count") { + scenario("analyze avg file count") { + Given("a.scala: 10 lines, b.scala: 10 lines, c.sbt: 5 lines, d: 5 lines") + When("calculte avg file count") + Then("result should be 7.5") codeBaseAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders).get.avgLineCount shouldBe 7.5 } - it("can find longest file") { + scenario("find longest file") { + Given("a.scala: 10 lines, b.scala: 5 lines") + When("finding longest file") + Then("a.scala should be returned") codeBaseAnalyzer.longestFile(List(aInfo, bInfo)) shouldBe aInfo } - it("will return top 10 longest files") { + scenario("find top 10 longest files") { + Given("11 files whose line of code is 1 to 11") val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) + When("finding top 10 longest files") val top10LongFiles = codeBaseAnalyzer.top10Files(sourceCodeInfos) + Then("10 files should be returned") top10LongFiles should have size 10 + And("result should not contains file whose line of code 1") top10LongFiles should not contain SourceCodeInfo("1.scala", "1.scala", 1) } - it("can count total line numbers") { + scenario("count total line numbers") { + Given("a.scala: 10 lines, b.scala: 5") + When("count total line numbers") + Then("total line numbers should be 15") codeBaseAnalyzer.totalLineCount(List(aInfo, bInfo)) shouldBe 15 } - it("when directory scanner returns empty, code analyzer should return None") { + scenario("when directory scanner returns empty, code analyzer should return None") { + Given("a directory contains no source code file") val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? } + When("analyze that empty directory") + Then("it should return None(instead of broken)") emptyCodeAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) shouldBe None } } From 28c83b3453457c6c907e63325aed182951f4207d Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 6 Mar 2017 00:53:44 +0800 Subject: [PATCH 069/116] add a placeholder for performance test --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index cde0a3b..4260208 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,11 @@ pipeline { sh 'sbt "testOnly * -- -n com.github.notyy.codeAnalyzer.FunctionalTest"' } } + stage('performance test') { + steps { + sh 'echo performance test' + } + } stage('assembly') { steps { sh 'sbt assembly' From bcf1a2adc942f97e3ed01a56a9bc1da4a61778db Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 6 Mar 2017 10:09:10 +0800 Subject: [PATCH 070/116] implement deploy commands --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4260208..7ceebe2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { } stage('deploy') { steps { - sh 'echo deploy' + sh 'cp target/scala-2.11/CodeAnalyzerTutorial-assembly-0.0.1.jar /Users/twer/dev/bin/CodeAnalyzer.jar' } } stage('health check') { From c510c033aaf6ca2c49bef8a6abc5c22c807f2b2d Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 6 Mar 2017 23:21:24 +0800 Subject: [PATCH 071/116] add some java tdd example --- src/main/java/codeDetector/CodeDetector.java | 14 ++++++++++++++ src/main/java/codeDetector/DetectorReport.java | 13 +++++++++++++ src/test/java/codeDetector/CodeDetectorTest.java | 15 +++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 src/main/java/codeDetector/CodeDetector.java create mode 100644 src/main/java/codeDetector/DetectorReport.java create mode 100644 src/test/java/codeDetector/CodeDetectorTest.java diff --git a/src/main/java/codeDetector/CodeDetector.java b/src/main/java/codeDetector/CodeDetector.java new file mode 100644 index 0000000..4bc13fb --- /dev/null +++ b/src/main/java/codeDetector/CodeDetector.java @@ -0,0 +1,14 @@ +package codeDetector; + +import java.io.File; +import java.util.Arrays; + +public class CodeDetector { + public DetectorReport analyze(String path) { + File file = new File(path); + long fileCount = Arrays.stream(file.listFiles()).filter(File::isFile).count(); + DetectorReport detectorReport = new DetectorReport(); + detectorReport.setFileCount(fileCount); + return detectorReport; + } +} diff --git a/src/main/java/codeDetector/DetectorReport.java b/src/main/java/codeDetector/DetectorReport.java new file mode 100644 index 0000000..c7cac1e --- /dev/null +++ b/src/main/java/codeDetector/DetectorReport.java @@ -0,0 +1,13 @@ +package codeDetector; + +public class DetectorReport { + private long fileCount; + + public long getFileCount() { + return fileCount; + } + + public void setFileCount(long fileCount) { + this.fileCount = fileCount; + } +} diff --git a/src/test/java/codeDetector/CodeDetectorTest.java b/src/test/java/codeDetector/CodeDetectorTest.java new file mode 100644 index 0000000..ecbdf25 --- /dev/null +++ b/src/test/java/codeDetector/CodeDetectorTest.java @@ -0,0 +1,15 @@ +package codeDetector; + +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +public class CodeDetectorTest { + @Test + public void can_tell_how_many_files_in_a_directory(){ + CodeDetector codeDetector = new CodeDetector(); + DetectorReport detectorReport = codeDetector.analyze("src/test/fixture"); + assertThat(detectorReport.getFileCount(), is(1L)); + } +} \ No newline at end of file From 699ac001033a61f9a665dad9ee6dbae446165d3e Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 9 Jun 2017 16:24:58 +0800 Subject: [PATCH 072/116] use max instead of sort.last --- src/main/scala/tutor/CodebaseAnalyzer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index d1f4b60..dce7080 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -36,7 +36,7 @@ trait CodebaseAnalyzer { } private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { - sourceCodeInfos.sorted.last + sourceCodeInfos.max } private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { From b8eccbf0b6916de342e3f8146409f360230447ac Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 9 Jun 2017 17:55:31 +0800 Subject: [PATCH 073/116] extract a CodebaseAnalyzeAggregator --- .../tutor/CodebaseAnalyzeAggregator.scala | 27 ++++++++ src/main/scala/tutor/CodebaseAnalyzer.scala | 20 +----- .../scala/tutor/CodebaseAnalyzerParImpl.scala | 2 +- .../scala/tutor/CodebaseAnalyzerSeqImpl.scala | 2 +- src/main/scala/tutor/MainApp.scala | 4 +- .../tutor/CodebaseAnalyzeAggregatorSpec.scala | 57 +++++++++++++++++ .../scala/tutor/CodebaseAnalyzerSpec.scala | 62 +++++++------------ 7 files changed, 111 insertions(+), 63 deletions(-) create mode 100644 src/main/scala/tutor/CodebaseAnalyzeAggregator.scala create mode 100644 src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala new file mode 100644 index 0000000..d9f9e5d --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala @@ -0,0 +1,27 @@ +package tutor +import tutor.utils.FileUtil +import tutor.utils.FileUtil.Path + +trait CodebaseAnalyzeAggregator { + + private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { + files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) + } + + private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { + sourceCodeInfos.max + } + + private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { + sourceCodeInfos.sortBy(_.count).reverse.take(10) + } + + private[tutor] def totalLineCount(sourceCodeInfos: Seq[SourceCodeInfo]): Int = { + sourceCodeInfos.map(_.count).sum + } + + def avgLines(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Double = { + val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + avgLineCount + } +} diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index dce7080..749bddf 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -9,7 +9,7 @@ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avg ) trait CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { val files = BenchmarkUtil.record("scan folders") { @@ -22,28 +22,12 @@ trait CodebaseAnalyzer { processSourceFiles(files) } BenchmarkUtil.record("make last result ##") { - val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + val avgLineCount: Double = avgLines(files, sourceCodeInfos) Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } } } - protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] - private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { - files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) - } - - private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { - sourceCodeInfos.max - } - - private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { - sourceCodeInfos.sortBy(_.count).reverse.take(10) - } - - private[tutor] def totalLineCount(sourceCodeInfos: Seq[SourceCodeInfo]): Int = { - sourceCodeInfos.map(_.count).sum - } } diff --git a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala index ff24d64..befa3ac 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerParImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector diff --git a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala index 8c6dce1..457265f 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerSeqImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.map(processFile).filter(_.isSuccess).map(_.get) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 0248adf..29e9440 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,10 +15,10 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val file = new File(path) val analyzer = args.find(_.startsWith("-p")).map { _ => logger.info("using par collection mode") - new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer + new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator }.getOrElse { logger.info("using sequence collection mode") - new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer + new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator } val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala new file mode 100644 index 0000000..7b3694e --- /dev/null +++ b/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala @@ -0,0 +1,57 @@ +package tutor + +import org.scalatest.{FeatureSpec, GivenWhenThen, ShouldMatchers} +import tutor.utils.FileUtil + +class CodebaseAnalyzeAggregatorSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { + + val codeBaseAnalyzeAggregator = new CodebaseAnalyzeAggregator {} + val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) + val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) + + feature("aggregate the result of individual source code analyze") { + scenario("count file numbers by type") { + Given("a folder contains 2 .scala file and a .sbt file and a file without ext") + val ls = List("a.scala", "b.scala", "c.sbt", "d") + When("analyze that folder") + Then("result should contain 2 scala file, 1 sbt file and 1 empty-type file") + codeBaseAnalyzeAggregator.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) + } + scenario("analyze avg file count") { + Given("a.scala: 10 lines, b.scala: 10 lines, c.sbt: 5 lines, d: 5 lines") + val ls = List("a.scala", "b.scala", "c.sbt", "d") + val sourceCodeInfos = List( + SourceCodeInfo("a.scala", "a.scala", 10), + SourceCodeInfo("b.scala", "b.scala", 10), + SourceCodeInfo("c.sbt", "c.sbt", 5), + SourceCodeInfo("d", "d", 5) + ) + When("calculte avg file count") + val avgLines = codeBaseAnalyzeAggregator.avgLines(ls, sourceCodeInfos) + Then("result should be 7.5") + avgLines shouldBe 7.5 + } + scenario("find longest file") { + Given("a.scala: 10 lines, b.scala: 5 lines") + When("finding longest file") + Then("a.scala should be returned") + codeBaseAnalyzeAggregator.longestFile(List(aInfo, bInfo)) shouldBe aInfo + } + scenario("find top 10 longest files") { + Given("11 files whose line of code is 1 to 11") + val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) + When("finding top 10 longest files") + val top10LongFiles = codeBaseAnalyzeAggregator.top10Files(sourceCodeInfos) + Then("10 files should be returned") + top10LongFiles should have size 10 + And("result should not contains file whose line of code 1") + top10LongFiles should not contain SourceCodeInfo("1.scala", "1.scala", 1) + } + scenario("count total line numbers") { + Given("a.scala: 10 lines, b.scala: 5") + When("count total line numbers") + Then("total line numbers should be 15") + codeBaseAnalyzeAggregator.totalLineCount(List(aInfo, bInfo)) shouldBe 15 + } + } +} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index e3a36cc..9854da3 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -8,7 +8,7 @@ import scala.util.{Success, Try} class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { - val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { + val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): Try[SourceCodeInfo] = path match { @@ -19,52 +19,14 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe } } - val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) - val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) - info("As a technical consultant") info("I want to analyze a customer's code base") info("So that I can find bad smell in code base more easily") - feature("Analyze a folder containing source code") { - scenario("count file numbers by type") { - Given("a folder contains 2 .scala file and a .sbt file and a file without ext") - val ls = List("a.scala", "b.scala", "c.sbt", "d") - When("analyze that folder") - Then("result should contain 2 scala file, 1 sbt file and 1 empty-type file") - codeBaseAnalyzer.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) - } - scenario("analyze avg file count") { - Given("a.scala: 10 lines, b.scala: 10 lines, c.sbt: 5 lines, d: 5 lines") - When("calculte avg file count") - Then("result should be 7.5") - codeBaseAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders).get.avgLineCount shouldBe 7.5 - } - scenario("find longest file") { - Given("a.scala: 10 lines, b.scala: 5 lines") - When("finding longest file") - Then("a.scala should be returned") - codeBaseAnalyzer.longestFile(List(aInfo, bInfo)) shouldBe aInfo - } - scenario("find top 10 longest files") { - Given("11 files whose line of code is 1 to 11") - val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) - When("finding top 10 longest files") - val top10LongFiles = codeBaseAnalyzer.top10Files(sourceCodeInfos) - Then("10 files should be returned") - top10LongFiles should have size 10 - And("result should not contains file whose line of code 1") - top10LongFiles should not contain SourceCodeInfo("1.scala", "1.scala", 1) - } - scenario("count total line numbers") { - Given("a.scala: 10 lines, b.scala: 5") - When("count total line numbers") - Then("total line numbers should be 15") - codeBaseAnalyzer.totalLineCount(List(aInfo, bInfo)) shouldBe 15 - } + feature("analyze source code folder and give statistic results") { scenario("when directory scanner returns empty, code analyzer should return None") { Given("a directory contains no source code file") - val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { + val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? @@ -73,5 +35,23 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe Then("it should return None(instead of broken)") emptyCodeAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) shouldBe None } + scenario("when analyze a folder with source code should return correct analyze result") { + Given("source code folder") + //use test/fixutre as test data + When("analyze the folder") + val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + val analyzeResult = codeAnalyzer.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) + Then("it should return correct result") + analyzeResult shouldBe 'defined + val codeBaseInfo = analyzeResult.get + codeBaseInfo.avgLineCount shouldBe 16.0 + val fileTypeNums = codeBaseInfo.fileTypeNums + fileTypeNums.keySet.size shouldBe 2 + fileTypeNums("scala") shouldBe 1 + fileTypeNums("java") shouldBe 1 + codeBaseInfo.longestFileInfo.localPath shouldBe "SomeCode.scala" + codeBaseInfo.top10Files.length shouldBe 2 + codeBaseInfo.totalLineCount shouldBe 32 + } } } From 4ccd0410d7dd8eba392d325ca2dd6f3d8316c58c Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 9 Jun 2017 17:58:31 +0800 Subject: [PATCH 074/116] add a FunctionalTest tag --- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 9854da3..c21ed1e 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -1,6 +1,7 @@ package tutor import org.scalatest.{FeatureSpec, FunSpec, GivenWhenThen, ShouldMatchers} +import tags.TestTypeTag.FunctionalTest import tutor.utils.FileUtil import tutor.utils.FileUtil.Path @@ -35,7 +36,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe Then("it should return None(instead of broken)") emptyCodeAnalyzer.analyze("anypath", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) shouldBe None } - scenario("when analyze a folder with source code should return correct analyze result") { + scenario("when analyze a folder with source code should return correct analyze result", FunctionalTest) { Given("source code folder") //use test/fixutre as test data When("analyze the folder") From 39c9297fcdc7b38e0c7d468c0a4e45439760066f Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 9 Jun 2017 18:19:03 +0800 Subject: [PATCH 075/116] refactor DirectoryScanner to make it accept a function to process file --- src/main/scala/tutor/DirectoryScanner.scala | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 27d836d..8269d47 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -16,17 +16,27 @@ trait DirectoryScanner extends StrictLogging { * @return */ def scan(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = { + scan(path)(Vector[Path](), ignoreFolders) { + (acc, f) => + val filePath = f.getAbsolutePath + if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { + acc :+ filePath + } else acc + } + } + + def scan[T](path: Path)(initValue: T, ignoreFolders: Set[String])(processFile: (T, File) => T): T = { val files = new File(path).listFiles() if (files == null) { logger.warn(s"$path is not a legal directory") - Vector[Path]() + initValue } else { - files.foldLeft(Vector[Path]()) { (acc, f) => - val filePath = f.getAbsolutePath - if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { - acc :+ filePath - } else if (f.isDirectory && (!ignoreFolders.contains(FileUtil.extractLocalPath(f.getPath)))) { - acc ++ scan(filePath, knownFileTypes, ignoreFolders) + files.foldLeft(initValue) { (acc, file) => + val filePath = file.getAbsolutePath + if (file.isFile) { + processFile(acc, file) + } else if (file.isDirectory && (!ignoreFolders.contains(FileUtil.extractLocalPath(file.getPath)))) { + scan(filePath)(acc, ignoreFolders)(processFile) } else { acc } From 77ddb5e664a0935ebddc7ef36c2f764abef24f77 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 12 Jun 2017 16:25:50 +0800 Subject: [PATCH 076/116] add test for par implementation --- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index c21ed1e..aa2559f 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -53,6 +53,10 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe codeBaseInfo.longestFileInfo.localPath shouldBe "SomeCode.scala" codeBaseInfo.top10Files.length shouldBe 2 codeBaseInfo.totalLineCount shouldBe 32 + //test par implementation + val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) + analyzeResult shouldBe analyzeResultOfPar } } } From 4d4abe5f8fdb86fbeefde48d76c3b67bf735d58c Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 12 Jun 2017 17:29:49 +0800 Subject: [PATCH 077/116] extract CodebaseAnalyzerInterface so that akka implementation can extends it --- src/main/scala/tutor/CodebaseAnalyzer.scala | 4 ++-- src/main/scala/tutor/CodebaseAnalyzerInterface.scala | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/tutor/CodebaseAnalyzerInterface.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 749bddf..9fcc4b1 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -8,10 +8,10 @@ case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avg top10Files: Seq[SourceCodeInfo] ) -trait CodebaseAnalyzer { +trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => - def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { + override def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { val files = BenchmarkUtil.record("scan folders") { scan(path, knownFileTypes, ignoreFolders) } diff --git a/src/main/scala/tutor/CodebaseAnalyzerInterface.scala b/src/main/scala/tutor/CodebaseAnalyzerInterface.scala new file mode 100644 index 0000000..9b13fb7 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerInterface.scala @@ -0,0 +1,7 @@ +package tutor +import tutor.utils.FileUtil.Path + +trait CodebaseAnalyzerInterface { + + def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] +} From 73cefeaf2eb732eb99ba576acfb699a03bf4f644 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 12 Jun 2017 17:30:38 +0800 Subject: [PATCH 078/116] extract more behavior to CodebaseAnalyzeAggregator --- src/main/scala/tutor/CodebaseAnalyzeAggregator.scala | 7 ++++++- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala index d9f9e5d..41adc60 100644 --- a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala @@ -20,8 +20,13 @@ trait CodebaseAnalyzeAggregator { sourceCodeInfos.map(_.count).sum } - def avgLines(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Double = { + private[tutor] def avgLines(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Double = { val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length avgLineCount } + + private[tutor] def aggregate(files:Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Option[CodebaseInfo]= { + val avgLineCount: Double = avgLines(files, sourceCodeInfos) + Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) + } } diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index aa2559f..9cf34e2 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -57,6 +57,10 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar + //test akka implementation +// val codeAnalyzerAkkaImpl = new CodebaseAnalyzerAkkaImpl with DirectoryScanner +// val analyzeResultOfAkka = codeAnalyzerAkkaImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) +// analyzeResult shouldBe analyzeResultOfAkka } } } From ad79639f5f1c2f8d59a139cac0da488f0499a6e9 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 13 Jun 2017 15:11:27 +0800 Subject: [PATCH 079/116] add a foreach to DirectoryScanner --- src/main/scala/tutor/DirectoryScanner.scala | 10 ++++++++++ src/test/scala/tutor/DirectoryScannerSpec.scala | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index 8269d47..dec0975 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -44,6 +44,16 @@ trait DirectoryScanner extends StrictLogging { } } + def foreachFile(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String])(processFile: File => Unit): Unit = { + scan(path)((), ignoreFolders) { + (acc, f) => + val filePath = f.getAbsolutePath + if (f.isFile && shouldAccept(f.getPath, knownFileTypes)) { + processFile(f) + } else () + } + } + private def shouldAccept(path: Path, knownFileTypes: Set[String]): Boolean = { knownFileTypes.contains(FileUtil.extractExtFileName(path)) } diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index 6ded74a..bb615cf 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -1,9 +1,13 @@ package tutor +import java.io.File + import org.scalatest.{FunSpec, ShouldMatchers} import tags.TestTypeTag.FunctionalTest import tutor.utils.FileUtil +import scala.collection.mutable.ArrayBuffer + class DirectoryScannerSpec extends FunSpec with ShouldMatchers { describe("DirectoryScanner") { it("can scan directory recursively and return all file paths" + @@ -13,5 +17,15 @@ class DirectoryScannerSpec extends FunSpec with ShouldMatchers { files.length shouldBe 2 FileUtil.extractLocalPath(files.head) shouldBe "SomeCode.scala" } + it("can scan directory recursively and let user do what they want on the file, " + + "and it should only accept known txt files", FunctionalTest){ + val ds = new DirectoryScanner {} + val files:ArrayBuffer[File] = new ArrayBuffer[File]() + ds.foreachFile("src/test/fixture", Set("scala", "java"), Set("target")){ file => + files += file + } + files.length shouldBe 2 + FileUtil.extractLocalPath(files.head.getAbsolutePath) shouldBe "SomeCode.scala" + } } } From 9fa124b076f270c97dffd5ebd7ff281f55990218 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 13 Jun 2017 18:22:58 +0800 Subject: [PATCH 080/116] change longestFileInfo to Optional --- .../scala/tutor/CodebaseAnalyzeAggregator.scala | 14 ++++++++------ src/main/scala/tutor/CodebaseAnalyzer.scala | 6 +++++- src/main/scala/tutor/ReportFormatter.scala | 8 ++++---- .../tutor/CodebaseAnalyzeAggregatorSpec.scala | 2 +- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 4 ++-- src/test/scala/tutor/ReportFormatterSpec.scala | 2 +- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala index 41adc60..7c48f17 100644 --- a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala @@ -1,4 +1,5 @@ package tutor + import tutor.utils.FileUtil import tutor.utils.FileUtil.Path @@ -8,24 +9,25 @@ trait CodebaseAnalyzeAggregator { files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) } - private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): SourceCodeInfo = { - sourceCodeInfos.max + private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): Option[SourceCodeInfo] = { + if (sourceCodeInfos.isEmpty) None + else Some(sourceCodeInfos.max) } private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { - sourceCodeInfos.sortBy(_.count).reverse.take(10) + sourceCodeInfos.sortBy(_.lineCount).reverse.take(10) } private[tutor] def totalLineCount(sourceCodeInfos: Seq[SourceCodeInfo]): Int = { - sourceCodeInfos.map(_.count).sum + sourceCodeInfos.map(_.lineCount).sum } private[tutor] def avgLines(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Double = { - val avgLineCount = sourceCodeInfos.map(_.count).sum.toDouble / files.length + val avgLineCount = sourceCodeInfos.map(_.lineCount).sum.toDouble / files.length avgLineCount } - private[tutor] def aggregate(files:Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Option[CodebaseInfo]= { + private[tutor] def aggregate(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Option[CodebaseInfo] = { val avgLineCount: Double = avgLines(files, sourceCodeInfos) Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) } diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 9fcc4b1..9a68f0a 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -3,8 +3,12 @@ package tutor import tutor.utils.{BenchmarkUtil, FileUtil} import tutor.utils.FileUtil._ +object CodebaseInfo { + def empty:CodebaseInfo = new CodebaseInfo(Map.empty[String,Int],0,0,null,Seq.empty[SourceCodeInfo]) +} + case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avgLineCount: Double, - longestFileInfo: SourceCodeInfo, + longestFileInfo: Option[SourceCodeInfo], top10Files: Seq[SourceCodeInfo] ) diff --git a/src/main/scala/tutor/ReportFormatter.scala b/src/main/scala/tutor/ReportFormatter.scala index ce312fc..f430fe4 100644 --- a/src/main/scala/tutor/ReportFormatter.scala +++ b/src/main/scala/tutor/ReportFormatter.scala @@ -2,7 +2,7 @@ package tutor trait ReportFormatter { def format(codebaseInfo: CodebaseInfo): String = { - val longestFileInfo: SourceCodeInfo = codebaseInfo.longestFileInfo + val longestFileInfo: Option[SourceCodeInfo] = codebaseInfo.longestFileInfo codebaseInfo.fileTypeNums.map { case (fileType, count) => s"$fileType $count" }.mkString("\n") ++ @@ -10,17 +10,17 @@ trait ReportFormatter { ReportFormatter.separator ++ "\n\n" ++ s"total line count: ${codebaseInfo.totalLineCount}" ++ "\n" ++ s"avg line count: ${codebaseInfo.avgLineCount}" ++ "\n" ++ - s"longest file: ${longestFileInfo.path} ${longestFileInfo.count}" ++ + s"longest file: ${longestFileInfo.map(_.path).getOrElse("not avaliable")} ${longestFileInfo.map(_.lineCount).getOrElse(0)}" ++ "\n" ++ ReportFormatter.separator ++ "\n\n" ++ "top 10 long files\n" ++ codebaseInfo.top10Files.map { - s => s"${s.path} ${s.count}" + s => s"${s.path} ${s.lineCount}" }.mkString("\n") } def format(sourceCode: SourceCodeInfo): String = { - s"name: ${sourceCode.localPath} lines: ${sourceCode.count}" + s"name: ${sourceCode.localPath} lines: ${sourceCode.lineCount}" } } diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala index 7b3694e..52dd837 100644 --- a/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala @@ -35,7 +35,7 @@ class CodebaseAnalyzeAggregatorSpec extends FeatureSpec with ShouldMatchers with Given("a.scala: 10 lines, b.scala: 5 lines") When("finding longest file") Then("a.scala should be returned") - codeBaseAnalyzeAggregator.longestFile(List(aInfo, bInfo)) shouldBe aInfo + codeBaseAnalyzeAggregator.longestFile(List(aInfo, bInfo)).get shouldBe aInfo } scenario("find top 10 longest files") { Given("11 files whose line of code is 1 to 11") diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 9cf34e2..d52fae4 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -50,14 +50,14 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe fileTypeNums.keySet.size shouldBe 2 fileTypeNums("scala") shouldBe 1 fileTypeNums("java") shouldBe 1 - codeBaseInfo.longestFileInfo.localPath shouldBe "SomeCode.scala" + codeBaseInfo.longestFileInfo.get.localPath shouldBe "SomeCode.scala" codeBaseInfo.top10Files.length shouldBe 2 codeBaseInfo.totalLineCount shouldBe 32 //test par implementation val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar - //test akka implementation +// test akka implementation // val codeAnalyzerAkkaImpl = new CodebaseAnalyzerAkkaImpl with DirectoryScanner // val analyzeResultOfAkka = codeAnalyzerAkkaImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) // analyzeResult shouldBe analyzeResultOfAkka diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 0ba6b51..894281e 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -15,7 +15,7 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), totalLineCount = 15, avgLineCount = 7.5, - SourceCodeInfo("absolute/a.scala", "a.scala", 10), { + Some(SourceCodeInfo("absolute/a.scala", "a.scala", 10)), { for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"absolute/$i.scala", s"$i.scala", i) } ) From 1f584a1a9f65a34c37e18ef289005aa8fdc869fb Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 13 Jun 2017 18:23:22 +0800 Subject: [PATCH 081/116] rename count to lineCount --- src/main/scala/tutor/SourceCodeInfo.scala | 4 ++-- src/test/scala/tutor/SourceCodeAnalyzerSpec.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index f56c6c9..b1b8881 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -6,12 +6,12 @@ import tutor.utils.FileUtil import scala.util.Try -case class SourceCodeInfo(path: String, localPath: String, count: Int) +final case class SourceCodeInfo(path: String, localPath: String, lineCount: Int) object SourceCodeInfo { implicit object SourceCodeInfoOrdering extends Ordering[SourceCodeInfo] { - override def compare(x: SourceCodeInfo, y: SourceCodeInfo): Int = x.count compare y.count + override def compare(x: SourceCodeInfo, y: SourceCodeInfo): Int = x.lineCount compare y.lineCount } } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 749e60b..4ef6d0b 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -10,7 +10,7 @@ class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers { val sourceCodeInfo = sca.processFile("./src/test/fixture/sourceFileSample").get sourceCodeInfo.localPath shouldBe "sourceFileSample" sourceCodeInfo.path shouldBe "./src/test/fixture/sourceFileSample" - sourceCodeInfo.count shouldBe 108 + sourceCodeInfo.lineCount shouldBe 108 } } } From e91e495eeec8218d5352b7162d3556212c189bf4 Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 14 Jun 2017 17:37:05 +0800 Subject: [PATCH 082/116] implement CodebaseInfo + SourceCodeInfo = CodebaseInfo, this make things much easier --- .../tutor/CodebaseAnalyzeAggregator.scala | 34 ----------- src/main/scala/tutor/CodebaseAnalyzer.scala | 33 ++++++++--- .../scala/tutor/CodebaseAnalyzerParImpl.scala | 2 +- .../scala/tutor/CodebaseAnalyzerSeqImpl.scala | 2 +- src/main/scala/tutor/MainApp.scala | 4 +- .../tutor/CodebaseAnalyzeAggregatorSpec.scala | 57 ------------------- .../scala/tutor/CodebaseAnalyzerSpec.scala | 8 +-- src/test/scala/tutor/CodebaseInfoSpec.scala | 20 +++++++ .../scala/tutor/ReportFormatterSpec.scala | 10 +--- 9 files changed, 56 insertions(+), 114 deletions(-) delete mode 100644 src/main/scala/tutor/CodebaseAnalyzeAggregator.scala delete mode 100644 src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala create mode 100644 src/test/scala/tutor/CodebaseInfoSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala deleted file mode 100644 index 7c48f17..0000000 --- a/src/main/scala/tutor/CodebaseAnalyzeAggregator.scala +++ /dev/null @@ -1,34 +0,0 @@ -package tutor - -import tutor.utils.FileUtil -import tutor.utils.FileUtil.Path - -trait CodebaseAnalyzeAggregator { - - private[tutor] def countFileTypeNum(files: Seq[Path]): Map[String, Int] = { - files.groupBy(FileUtil.extractExtFileName).mapValues(_.length) - } - - private[tutor] def longestFile(sourceCodeInfos: Seq[SourceCodeInfo]): Option[SourceCodeInfo] = { - if (sourceCodeInfos.isEmpty) None - else Some(sourceCodeInfos.max) - } - - private[tutor] def top10Files(sourceCodeInfos: Seq[SourceCodeInfo]): Seq[SourceCodeInfo] = { - sourceCodeInfos.sortBy(_.lineCount).reverse.take(10) - } - - private[tutor] def totalLineCount(sourceCodeInfos: Seq[SourceCodeInfo]): Int = { - sourceCodeInfos.map(_.lineCount).sum - } - - private[tutor] def avgLines(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Double = { - val avgLineCount = sourceCodeInfos.map(_.lineCount).sum.toDouble / files.length - avgLineCount - } - - private[tutor] def aggregate(files: Seq[Path], sourceCodeInfos: Seq[SourceCodeInfo]): Option[CodebaseInfo] = { - val avgLineCount: Double = avgLines(files, sourceCodeInfos) - Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) - } -} diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 9a68f0a..6ad264c 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -2,18 +2,36 @@ package tutor import tutor.utils.{BenchmarkUtil, FileUtil} import tutor.utils.FileUtil._ +import scala.math.max object CodebaseInfo { - def empty:CodebaseInfo = new CodebaseInfo(Map.empty[String,Int],0,0,null,Seq.empty[SourceCodeInfo]) + def empty: CodebaseInfo = new CodebaseInfo(0, Map.empty[String, Int], 0, 0, None, Seq.empty[SourceCodeInfo]) } -case class CodebaseInfo(fileTypeNums: Map[String, Int], totalLineCount: Int, avgLineCount: Double, - longestFileInfo: Option[SourceCodeInfo], - top10Files: Seq[SourceCodeInfo] - ) +case class CodebaseInfo(totalFileNums: Int, fileTypeNums: Map[String, Int], totalLineCount: Int, avgLineCount: Double, longestFileInfo: Option[SourceCodeInfo], top10Files: Seq[SourceCodeInfo]) { + def +(sourceCodeInfo: SourceCodeInfo): CodebaseInfo = { + val fileExt = FileUtil.extractExtFileName(sourceCodeInfo.localPath) + val newFileTypeNums: Map[String, Int] = if (fileTypeNums.contains(fileExt)) { + fileTypeNums.updated(fileExt, fileTypeNums(fileExt) + 1) + } else { + fileTypeNums + (fileExt -> 1) + } + val newTotalLineCount = totalLineCount + sourceCodeInfo.lineCount + val newTotalFileNum = totalFileNums + 1 + CodebaseInfo(newTotalFileNum,newFileTypeNums,newTotalLineCount, newTotalLineCount / newTotalFileNum, + if(longestFileInfo.isEmpty) { + Some(sourceCodeInfo) + }else{ + if(longestFileInfo.get.lineCount < sourceCodeInfo.lineCount) Some(sourceCodeInfo) + else longestFileInfo + }, + (top10Files :+ sourceCodeInfo).sortBy(_.lineCount).reverse.take(10) + ) + } +} trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { - this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => + this: DirectoryScanner with SourceCodeAnalyzer => override def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { val files = BenchmarkUtil.record("scan folders") { @@ -26,8 +44,7 @@ trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { processSourceFiles(files) } BenchmarkUtil.record("make last result ##") { - val avgLineCount: Double = avgLines(files, sourceCodeInfos) - Some(CodebaseInfo(countFileTypeNum(files), totalLineCount(sourceCodeInfos), avgLineCount, longestFile(sourceCodeInfos), top10Files(sourceCodeInfos))) + Some(sourceCodeInfos.foldLeft(CodebaseInfo.empty)(_ + _)) } } } diff --git a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala index befa3ac..ff24d64 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerParImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => + this: DirectoryScanner with SourceCodeAnalyzer => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector diff --git a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala index 457265f..8c6dce1 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerSeqImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator => + this: DirectoryScanner with SourceCodeAnalyzer => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.map(processFile).filter(_.isSuccess).map(_.get) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 29e9440..0248adf 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,10 +15,10 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val file = new File(path) val analyzer = args.find(_.startsWith("-p")).map { _ => logger.info("using par collection mode") - new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer }.getOrElse { logger.info("using sequence collection mode") - new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer } val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala deleted file mode 100644 index 52dd837..0000000 --- a/src/test/scala/tutor/CodebaseAnalyzeAggregatorSpec.scala +++ /dev/null @@ -1,57 +0,0 @@ -package tutor - -import org.scalatest.{FeatureSpec, GivenWhenThen, ShouldMatchers} -import tutor.utils.FileUtil - -class CodebaseAnalyzeAggregatorSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { - - val codeBaseAnalyzeAggregator = new CodebaseAnalyzeAggregator {} - val aInfo: SourceCodeInfo = SourceCodeInfo("a.scala", "a.scala", 10) - val bInfo: SourceCodeInfo = SourceCodeInfo("b.scala", "b.scala", 5) - - feature("aggregate the result of individual source code analyze") { - scenario("count file numbers by type") { - Given("a folder contains 2 .scala file and a .sbt file and a file without ext") - val ls = List("a.scala", "b.scala", "c.sbt", "d") - When("analyze that folder") - Then("result should contain 2 scala file, 1 sbt file and 1 empty-type file") - codeBaseAnalyzeAggregator.countFileTypeNum(ls) should contain theSameElementsAs Map[String, Int](("scala", 2), (FileUtil.EmptyFileType, 1), ("sbt", 1)) - } - scenario("analyze avg file count") { - Given("a.scala: 10 lines, b.scala: 10 lines, c.sbt: 5 lines, d: 5 lines") - val ls = List("a.scala", "b.scala", "c.sbt", "d") - val sourceCodeInfos = List( - SourceCodeInfo("a.scala", "a.scala", 10), - SourceCodeInfo("b.scala", "b.scala", 10), - SourceCodeInfo("c.sbt", "c.sbt", 5), - SourceCodeInfo("d", "d", 5) - ) - When("calculte avg file count") - val avgLines = codeBaseAnalyzeAggregator.avgLines(ls, sourceCodeInfos) - Then("result should be 7.5") - avgLines shouldBe 7.5 - } - scenario("find longest file") { - Given("a.scala: 10 lines, b.scala: 5 lines") - When("finding longest file") - Then("a.scala should be returned") - codeBaseAnalyzeAggregator.longestFile(List(aInfo, bInfo)).get shouldBe aInfo - } - scenario("find top 10 longest files") { - Given("11 files whose line of code is 1 to 11") - val sourceCodeInfos = for (i <- 1 to 11) yield SourceCodeInfo(s"$i.scala", s"$i.scala", i) - When("finding top 10 longest files") - val top10LongFiles = codeBaseAnalyzeAggregator.top10Files(sourceCodeInfos) - Then("10 files should be returned") - top10LongFiles should have size 10 - And("result should not contains file whose line of code 1") - top10LongFiles should not contain SourceCodeInfo("1.scala", "1.scala", 1) - } - scenario("count total line numbers") { - Given("a.scala: 10 lines, b.scala: 5") - When("count total line numbers") - Then("total line numbers should be 15") - codeBaseAnalyzeAggregator.totalLineCount(List(aInfo, bInfo)) shouldBe 15 - } - } -} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index d52fae4..00b879f 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -9,7 +9,7 @@ import scala.util.{Success, Try} class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { - val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator { + val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): Try[SourceCodeInfo] = path match { @@ -27,7 +27,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe feature("analyze source code folder and give statistic results") { scenario("when directory scanner returns empty, code analyzer should return None") { Given("a directory contains no source code file") - val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator { + val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? @@ -40,7 +40,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe Given("source code folder") //use test/fixutre as test data When("analyze the folder") - val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer val analyzeResult = codeAnalyzer.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) Then("it should return correct result") analyzeResult shouldBe 'defined @@ -54,7 +54,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe codeBaseInfo.top10Files.length shouldBe 2 codeBaseInfo.totalLineCount shouldBe 32 //test par implementation - val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with CodebaseAnalyzeAggregator + val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar // test akka implementation diff --git a/src/test/scala/tutor/CodebaseInfoSpec.scala b/src/test/scala/tutor/CodebaseInfoSpec.scala new file mode 100644 index 0000000..e05af5c --- /dev/null +++ b/src/test/scala/tutor/CodebaseInfoSpec.scala @@ -0,0 +1,20 @@ +package tutor + +import org.scalatest.{FunSpec, ShouldMatchers} + +class CodebaseInfoSpec extends FunSpec with ShouldMatchers{ + describe("codeBaseInfo"){ + it("empty CodeBaseInfo + SourceCodeInfo = CodeBaseInfo(contains this SourceCodeInfo)"){ + val sourceCodeInfo = SourceCodeInfo("1.scala", "1.scala", 10) + val result = CodebaseInfo.empty + sourceCodeInfo + result.totalFileNums shouldBe 1 + result.totalLineCount shouldBe 10 + result.top10Files shouldBe Seq(sourceCodeInfo) + result.longestFileInfo.get shouldBe sourceCodeInfo + result.fileTypeNums.keySet.size shouldBe 1 + result.fileTypeNums.keySet should contain("scala") + result.fileTypeNums("scala") shouldBe 1 + result.avgLineCount shouldBe 10 + } + } +} diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 894281e..8ba5b85 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -12,13 +12,9 @@ class ReportFormatterSpec extends FunSpec with ShouldMatchers { rf.format(SourceCodeInfo("somepath", "some name", 10)) shouldBe "name: some name lines: 10" } it("can format CodebaseInfo") { - val codebaseInfo = CodebaseInfo(Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), - totalLineCount = 15, - avgLineCount = 7.5, - Some(SourceCodeInfo("absolute/a.scala", "a.scala", 10)), { - for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"absolute/$i.scala", s"$i.scala", i) - } - ) + val codebaseInfo = CodebaseInfo(0, Map("sbt" -> 1, "scala" -> 2, FileUtil.EmptyFileType -> 1), totalLineCount = 15, avgLineCount = 7.5, Some(SourceCodeInfo("absolute/a.scala", "a.scala", 10)), { + for (i <- 10 to 1 by -1) yield SourceCodeInfo(s"absolute/$i.scala", s"$i.scala", i) + }) rf.format(codebaseInfo) shouldBe s""" |sbt 1 From 6857ff4c012d863c7dcedfb2ea68d181aa03b141 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 15 Jun 2017 17:04:08 +0800 Subject: [PATCH 083/116] add more time recording util method --- src/main/scala/tutor/utils/BenchmarkUtil.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/scala/tutor/utils/BenchmarkUtil.scala b/src/main/scala/tutor/utils/BenchmarkUtil.scala index f811baa..40542b0 100644 --- a/src/main/scala/tutor/utils/BenchmarkUtil.scala +++ b/src/main/scala/tutor/utils/BenchmarkUtil.scala @@ -17,4 +17,16 @@ object BenchmarkUtil extends StrictLogging { logger.info(s"$actionDesc total elapsed ${sdf.format(elapsed)}") rs } + def recordStart(actionDesc: String):Date = { + logger.info(s"$actionDesc begin") + new Date + } + + def recordElapse(actionDesc: String, beginFrom: Date):Unit = { + logger.info(s"$actionDesc ended") + val endTime = new Date + val elapsed = new Date(endTime.getTime - beginFrom.getTime) + val sdf = new SimpleDateFormat("mm:ss.SSS") + logger.info(s"$actionDesc total elapsed ${sdf.format(elapsed)}") + } } From a12aade3c545e577f9a9f5cb7c9333ba92cbd6af Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 15 Jun 2017 17:07:12 +0800 Subject: [PATCH 084/116] implement an akka version --- .../CodebaseAnalyzeAggregatorActor.scala | 85 +++++++++++++++++++ .../scala/tutor/CodebaseAnalyzerAkkaApp.scala | 28 ++++++ .../CodebaseAnalyzerControllerActor.scala | 19 +++++ .../scala/tutor/SourceCodeAnalyzerActor.scala | 22 +++++ src/test/fixture/sub/sub1/othercode.java | 1 - .../CodebaseAnalyzeAggregatorActorSpec.scala | 29 +++++++ .../scala/tutor/CodebaseAnalyzerSpec.scala | 8 +- .../tutor/SourceCodeAnalyzerActorSpec.scala | 22 +++++ 8 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala create mode 100644 src/main/scala/tutor/CodebaseAnalyzerAkkaApp.scala create mode 100644 src/main/scala/tutor/CodebaseAnalyzerControllerActor.scala create mode 100644 src/main/scala/tutor/SourceCodeAnalyzerActor.scala create mode 100644 src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala create mode 100644 src/test/scala/tutor/SourceCodeAnalyzerActorSpec.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala new file mode 100644 index 0000000..64f5292 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala @@ -0,0 +1,85 @@ +package tutor + +import java.util.Date + +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props} +import akka.routing.{ActorRefRoutee, RoundRobinRoutingLogic, Router} +import tutor.CodebaseAnalyzeAggregatorActor.{AnalyzeDirectory, Complete, Report, Timeout} +import tutor.SourceCodeAnalyzerActor.NewFile +import tutor.utils.BenchmarkUtil + +import scala.concurrent.duration._ +import scala.util.{Failure, Success, Try} + +object CodebaseAnalyzeAggregatorActor { + def props(): Props = Props(new CodebaseAnalyzeAggregatorActor) + + final case class AnalyzeDirectory(path: String) + + final case class Complete(result: Try[SourceCodeInfo]) + + final case object Timeout + + final case class Report(codebaseInfo: CodebaseInfo) + +} + +class CodebaseAnalyzeAggregatorActor extends Actor with ActorLogging with DirectoryScanner with ReportFormatter { + var controller: ActorRef = _ + var currentPath: String = _ + var beginTime: Date = _ + var fileCount = 0 + var completeCount = 0 + var failCount = 0 + var result: CodebaseInfo = CodebaseInfo.empty + var timeoutTimer: Cancellable = _ + + var router: Router = { + val routees = Vector.fill(8) { + val r = context.actorOf(SourceCodeAnalyzerActor.props()) + context watch r + ActorRefRoutee(r) + } + Router(RoundRobinRoutingLogic(), routees) + } + + override def receive: Receive = { + case AnalyzeDirectory(path) => { + controller = sender() + currentPath = path + beginTime = BenchmarkUtil.recordStart(s"analyze folder $currentPath") + foreachFile(path, PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) { file => + fileCount += 1 + router.route(NewFile(file.getAbsolutePath), context.self) + } + import context.dispatcher + timeoutTimer = context.system.scheduler.scheduleOnce(3.seconds, context.self, Timeout) + } + case Complete(Success(sourceCodeInfo: SourceCodeInfo)) => { + completeCount += 1 + result = result + sourceCodeInfo + finishIfAllComplete() + } + case Complete(Failure(exception)) => { + completeCount += 1 + failCount += 1 + log.warning("processing file failed", exception) + finishIfAllComplete() + } + case Timeout => { + println(s"${result.totalFileNums} of $fileCount files processed before timeout") + controller ! Report(result) + BenchmarkUtil.recordElapse(s"analyze folder $currentPath", beginTime) + } + case x@_ => log.error(s"receive unknown message $x") + } + + def finishIfAllComplete(): Unit = { + if (completeCount == fileCount) { + timeoutTimer.cancel() + controller ! Report(result) + BenchmarkUtil.recordElapse(s"analyze folder $currentPath", beginTime) + context.stop(self) + } + } +} diff --git a/src/main/scala/tutor/CodebaseAnalyzerAkkaApp.scala b/src/main/scala/tutor/CodebaseAnalyzerAkkaApp.scala new file mode 100644 index 0000000..76c9e70 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerAkkaApp.scala @@ -0,0 +1,28 @@ +package tutor + +import akka.actor.{ActorRef, ActorSystem} +import tutor.CodebaseAnalyzeAggregatorActor.AnalyzeDirectory + +import scala.io.StdIn + +object CodebaseAnalyzerAkkaApp extends App { + + val system = ActorSystem("CodebaseAnalyzer") + val codebaseAnalyzerControllerActor: ActorRef = system.actorOf(CodebaseAnalyzerControllerActor.props()) + + var shouldContinue = true + try { + while (shouldContinue) { + println("please input source file folder or :q to quit") + val input = StdIn.readLine() + if (input == ":q") { + shouldContinue = false + } else { + codebaseAnalyzerControllerActor ! AnalyzeDirectory(input) + } + } + } finally { + println("good bye!") + system.terminate() + } +} diff --git a/src/main/scala/tutor/CodebaseAnalyzerControllerActor.scala b/src/main/scala/tutor/CodebaseAnalyzerControllerActor.scala new file mode 100644 index 0000000..9da5f4a --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerControllerActor.scala @@ -0,0 +1,19 @@ +package tutor + +import akka.actor.{Actor, Props} +import tutor.CodebaseAnalyzeAggregatorActor.{AnalyzeDirectory, Report} + +object CodebaseAnalyzerControllerActor { + def props(): Props = Props(new CodebaseAnalyzerControllerActor) +} + +class CodebaseAnalyzerControllerActor extends Actor with ReportFormatter { + override def receive: Receive = { + case AnalyzeDirectory(path) => { + context.actorOf(CodebaseAnalyzeAggregatorActor.props()) ! AnalyzeDirectory(path) + } + case Report(content) => { + println(format(content)) + } + } +} diff --git a/src/main/scala/tutor/SourceCodeAnalyzerActor.scala b/src/main/scala/tutor/SourceCodeAnalyzerActor.scala new file mode 100644 index 0000000..09c4b50 --- /dev/null +++ b/src/main/scala/tutor/SourceCodeAnalyzerActor.scala @@ -0,0 +1,22 @@ +package tutor + +import akka.actor.{Actor, ActorLogging, Props} +import tutor.CodebaseAnalyzeAggregatorActor.Complete +import tutor.SourceCodeAnalyzerActor.NewFile + + +object SourceCodeAnalyzerActor { + def props(): Props = Props(new SourceCodeAnalyzerActor) + + final case class NewFile(path: String) + +} + +class SourceCodeAnalyzerActor extends Actor with ActorLogging with SourceCodeAnalyzer { + override def receive: Receive = { + case NewFile(path) => { + val sourceCodeInfo = processFile(path) + sender() ! Complete(sourceCodeInfo) + } + } +} diff --git a/src/test/fixture/sub/sub1/othercode.java b/src/test/fixture/sub/sub1/othercode.java index f448649..5011279 100644 --- a/src/test/fixture/sub/sub1/othercode.java +++ b/src/test/fixture/sub/sub1/othercode.java @@ -12,5 +12,4 @@ def fromFile(path: Path): SourceCodeInfo = { val lines = source.getLines.toList new SourceCodeInfo(path, extractLocalPath(path), lines) } - } \ No newline at end of file diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala new file mode 100644 index 0000000..a6687e6 --- /dev/null +++ b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala @@ -0,0 +1,29 @@ +package tutor + +import akka.actor.ActorSystem +import akka.testkit.TestProbe +import org.scalatest.{FunSpec, Matchers, ShouldMatchers} +import tutor.CodebaseAnalyzeAggregatorActor.{AnalyzeDirectory, Report} + +import scala.concurrent.duration._ + +class CodebaseAnalyzeAggregatorActorSpec extends FunSpec with ShouldMatchers { + describe("CodebaseAnalyzeAggregatorActor") { + it("can analyze given file path, aggregate results of all individual files") { + implicit val system = ActorSystem("CodebaseAnalyzeAggregator") + val probe = TestProbe() + val codebaseAnalyzeAggregator = system.actorOf(CodebaseAnalyzeAggregatorActor.props()) + codebaseAnalyzeAggregator.tell(AnalyzeDirectory("src/test/fixture"), probe.ref) + val result = probe.expectMsgType[Report](3 seconds).codebaseInfo + result.totalFileNums shouldBe 2 + result.fileTypeNums.keySet should have size 2 + result.fileTypeNums("java") shouldBe 1 + result.fileTypeNums("scala") shouldBe 1 + result.totalLineCount shouldBe 31 + result.avgLineCount shouldBe 15.0 + result.longestFileInfo.get.localPath shouldBe "SomeCode.scala" + result.top10Files should have size 2 + result.top10Files should contain (SourceCodeInfo("/Users/twer/source/scala/CodeAnalyzerTutorial/src/test/fixture/sub/SomeCode.scala", "SomeCode.scala", 16)) + } + } +} diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 00b879f..8b61e76 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -45,22 +45,18 @@ class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhe Then("it should return correct result") analyzeResult shouldBe 'defined val codeBaseInfo = analyzeResult.get - codeBaseInfo.avgLineCount shouldBe 16.0 + codeBaseInfo.avgLineCount shouldBe 15.0 val fileTypeNums = codeBaseInfo.fileTypeNums fileTypeNums.keySet.size shouldBe 2 fileTypeNums("scala") shouldBe 1 fileTypeNums("java") shouldBe 1 codeBaseInfo.longestFileInfo.get.localPath shouldBe "SomeCode.scala" codeBaseInfo.top10Files.length shouldBe 2 - codeBaseInfo.totalLineCount shouldBe 32 + codeBaseInfo.totalLineCount shouldBe 31 //test par implementation val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar -// test akka implementation -// val codeAnalyzerAkkaImpl = new CodebaseAnalyzerAkkaImpl with DirectoryScanner -// val analyzeResultOfAkka = codeAnalyzerAkkaImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) -// analyzeResult shouldBe analyzeResultOfAkka } } } diff --git a/src/test/scala/tutor/SourceCodeAnalyzerActorSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerActorSpec.scala new file mode 100644 index 0000000..63c1fd3 --- /dev/null +++ b/src/test/scala/tutor/SourceCodeAnalyzerActorSpec.scala @@ -0,0 +1,22 @@ +package tutor + +import akka.actor.ActorSystem +import akka.testkit.TestProbe +import org.scalatest.FunSpec +import tutor.CodebaseAnalyzeAggregatorActor.Complete +import tutor.SourceCodeAnalyzerActor.NewFile + +import scala.util.Success + +class SourceCodeAnalyzerActorSpec extends FunSpec { + describe("SourceCodeAnalyzerActor"){ + it("can analyze given file path, and reply with SourceCodeInfo"){ + implicit val system = ActorSystem("SourceCodeAnalyzer") + val probe = TestProbe() + val sourceCodeAnalyzerActor = system.actorOf(SourceCodeAnalyzerActor.props()) + sourceCodeAnalyzerActor.tell(NewFile("src/test/fixture/sub/SomeCode.scala"), probe.ref) + probe.expectMsg(Complete(Success(SourceCodeInfo(path = "src/test/fixture/sub/SomeCode.scala", + localPath = "SomeCode.scala", 16)))) + } + } +} From abd118d1790ba9be272872f2dae6a2e64a2f5378 Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 16 Jun 2017 11:30:34 +0800 Subject: [PATCH 085/116] adjust timeout to a dynamic value related to total file count --- src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala index 64f5292..e087786 100644 --- a/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala @@ -53,7 +53,7 @@ class CodebaseAnalyzeAggregatorActor extends Actor with ActorLogging with Direct router.route(NewFile(file.getAbsolutePath), context.self) } import context.dispatcher - timeoutTimer = context.system.scheduler.scheduleOnce(3.seconds, context.self, Timeout) + timeoutTimer = context.system.scheduler.scheduleOnce((fileCount / 1000).seconds, context.self, Timeout) } case Complete(Success(sourceCodeInfo: SourceCodeInfo)) => { completeCount += 1 @@ -63,7 +63,7 @@ class CodebaseAnalyzeAggregatorActor extends Actor with ActorLogging with Direct case Complete(Failure(exception)) => { completeCount += 1 failCount += 1 - log.warning("processing file failed", exception) + log.warning("processing file failed {}", exception) finishIfAllComplete() } case Timeout => { From 487eb7cb3ebca8787fc7a3561dfa214f27012e3b Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 16 Jun 2017 11:31:00 +0800 Subject: [PATCH 086/116] make error message more useful --- src/main/scala/tutor/SourceCodeInfo.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index b1b8881..8c5b310 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -24,7 +24,9 @@ trait SourceCodeAnalyzer extends StrictLogging { try { val lines = source.getLines.toList SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) - }finally { + } catch { + case e => throw new IllegalArgumentException(s"error processing file $path", e) + } finally { source.close() } } From c48abf5560106b23710c0d74c502907c2b0eb0f7 Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 16 Jun 2017 14:50:03 +0800 Subject: [PATCH 087/116] clean code --- src/main/scala/tutor/SourceCodeInfo.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index 8c5b310..bdaf748 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -25,7 +25,7 @@ trait SourceCodeAnalyzer extends StrictLogging { val lines = source.getLines.toList SourceCodeInfo(path, FileUtil.extractLocalPath(path), lines.length) } catch { - case e => throw new IllegalArgumentException(s"error processing file $path", e) + case e: Throwable => throw new IllegalArgumentException(s"error processing file $path", e) } finally { source.close() } From 03175a9573bb0f4a5c5f0e64ec34e1e6c58cb5cc Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 16 Jun 2017 14:51:22 +0800 Subject: [PATCH 088/116] optimize unnecessary sort --- src/main/scala/tutor/CodebaseAnalyzer.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 6ad264c..573b4dd 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -18,14 +18,20 @@ case class CodebaseInfo(totalFileNums: Int, fileTypeNums: Map[String, Int], tota } val newTotalLineCount = totalLineCount + sourceCodeInfo.lineCount val newTotalFileNum = totalFileNums + 1 - CodebaseInfo(newTotalFileNum,newFileTypeNums,newTotalLineCount, newTotalLineCount / newTotalFileNum, - if(longestFileInfo.isEmpty) { + CodebaseInfo(newTotalFileNum, newFileTypeNums, newTotalLineCount, newTotalLineCount / newTotalFileNum, + if (longestFileInfo.isEmpty) { Some(sourceCodeInfo) - }else{ - if(longestFileInfo.get.lineCount < sourceCodeInfo.lineCount) Some(sourceCodeInfo) + } else { + if (longestFileInfo.get.lineCount < sourceCodeInfo.lineCount) Some(sourceCodeInfo) else longestFileInfo }, - (top10Files :+ sourceCodeInfo).sortBy(_.lineCount).reverse.take(10) + if (top10Files.isEmpty) { + Vector(sourceCodeInfo) + } else if (top10Files.size < 10 || sourceCodeInfo.lineCount > top10Files.last.lineCount) { + (top10Files :+ sourceCodeInfo).sortBy(_.lineCount).reverse.take(10) + } else { + top10Files + } ) } } From b3680a50bf97afcb56812b2d99523aa75b3e7a70 Mon Sep 17 00:00:00 2001 From: notyy Date: Fri, 16 Jun 2017 16:03:36 +0800 Subject: [PATCH 089/116] upgrade scalatest version to 3.0.1 and akka version to 2.5.2 --- build.sbt | 33 ++++++++++++------- .../CodebaseAnalyzeAggregatorActorSpec.scala | 4 +-- .../scala/tutor/CodebaseAnalyzerSpec.scala | 7 ++-- src/test/scala/tutor/CodebaseInfoSpec.scala | 4 +-- .../scala/tutor/DirectoryScannerSpec.scala | 4 +-- .../scala/tutor/ReportFormatterSpec.scala | 4 +-- .../scala/tutor/SourceCodeAnalyzerSpec.scala | 4 +-- src/test/scala/tutor/utils/FileUtilSpec.scala | 4 +-- 8 files changed, 36 insertions(+), 28 deletions(-) diff --git a/build.sbt b/build.sbt index 36e1b4e..98ebc52 100644 --- a/build.sbt +++ b/build.sbt @@ -12,21 +12,30 @@ scalaVersion := "2.11.8" libraryDependencies ++= Seq( "org.scalacheck" %% "scalacheck" % "1.13.2" % "test", "org.pegdown" % "pegdown" % "1.0.2" % "test", //used in html report - "org.scalatest" %% "scalatest" % "2.2.1" % "test", + "org.scalatest" %% "scalatest" % "3.0.1" % "test", "org.slf4j" % "slf4j-api" % "1.7.7", "ch.qos.logback" % "logback-classic" % "1.1.2", "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", - "com.typesafe.akka" %% "akka-actor" % "2.4.7", - "com.typesafe.akka" %% "akka-http-core" % "2.4.7", - "com.typesafe.akka" %% "akka-http-testkit" % "2.4.7", - "com.typesafe.akka" %% "akka-persistence" % "2.4.7", - "com.typesafe.akka" %% "akka-persistence-tck" % "2.4.7", - "com.typesafe.akka" %% "akka-slf4j" % "2.4.7", - "com.typesafe.akka" %% "akka-stream" % "2.4.7", - "com.typesafe.akka" %% "akka-stream-testkit" % "2.4.7", - "com.typesafe.akka" %% "akka-testkit" % "2.4.7", - "com.typesafe.akka" %% "akka-http-experimental" % "2.4.7", - "com.typesafe.akka" %% "akka-http-jackson-experimental" % "2.4.7" + "com.typesafe.akka" %% "akka-actor" % "2.5.2", + "com.typesafe.akka" %% "akka-agent" % "2.5.2", + "com.typesafe.akka" %% "akka-camel" % "2.5.2", + "com.typesafe.akka" %% "akka-cluster" % "2.5.2", + "com.typesafe.akka" %% "akka-cluster-metrics" % "2.5.2", + "com.typesafe.akka" %% "akka-cluster-sharding" % "2.5.2", + "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.2", + "com.typesafe.akka" %% "akka-distributed-data" % "2.5.2", + "com.typesafe.akka" %% "akka-multi-node-testkit" % "2.5.2", + "com.typesafe.akka" %% "akka-osgi" % "2.5.2", + "com.typesafe.akka" %% "akka-persistence" % "2.5.2", + "com.typesafe.akka" %% "akka-persistence-query" % "2.5.2", + "com.typesafe.akka" %% "akka-persistence-tck" % "2.5.2", + "com.typesafe.akka" %% "akka-remote" % "2.5.2", + "com.typesafe.akka" %% "akka-slf4j" % "2.5.2", + "com.typesafe.akka" %% "akka-stream" % "2.5.2", + "com.typesafe.akka" %% "akka-stream-testkit" % "2.5.2", + "com.typesafe.akka" %% "akka-testkit" % "2.5.2", + "com.typesafe.akka" %% "akka-typed" % "2.5.2", + "com.typesafe.akka" %% "akka-contrib" % "2.5.2" ) // TODO reopen it later diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala index a6687e6..4329025 100644 --- a/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala @@ -2,12 +2,12 @@ package tutor import akka.actor.ActorSystem import akka.testkit.TestProbe -import org.scalatest.{FunSpec, Matchers, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} import tutor.CodebaseAnalyzeAggregatorActor.{AnalyzeDirectory, Report} import scala.concurrent.duration._ -class CodebaseAnalyzeAggregatorActorSpec extends FunSpec with ShouldMatchers { +class CodebaseAnalyzeAggregatorActorSpec extends FunSpec with Matchers { describe("CodebaseAnalyzeAggregatorActor") { it("can analyze given file path, aggregate results of all individual files") { implicit val system = ActorSystem("CodebaseAnalyzeAggregator") diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 8b61e76..40c31cb 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -1,13 +1,12 @@ package tutor -import org.scalatest.{FeatureSpec, FunSpec, GivenWhenThen, ShouldMatchers} -import tags.TestTypeTag.FunctionalTest -import tutor.utils.FileUtil +import _root_.tags.TestTypeTag.FunctionalTest +import org.scalatest._ import tutor.utils.FileUtil.Path import scala.util.{Success, Try} -class CodebaseAnalyzerSpec extends FeatureSpec with ShouldMatchers with GivenWhenThen { +class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen { val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") diff --git a/src/test/scala/tutor/CodebaseInfoSpec.scala b/src/test/scala/tutor/CodebaseInfoSpec.scala index e05af5c..23ac300 100644 --- a/src/test/scala/tutor/CodebaseInfoSpec.scala +++ b/src/test/scala/tutor/CodebaseInfoSpec.scala @@ -1,8 +1,8 @@ package tutor -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} -class CodebaseInfoSpec extends FunSpec with ShouldMatchers{ +class CodebaseInfoSpec extends FunSpec with Matchers{ describe("codeBaseInfo"){ it("empty CodeBaseInfo + SourceCodeInfo = CodeBaseInfo(contains this SourceCodeInfo)"){ val sourceCodeInfo = SourceCodeInfo("1.scala", "1.scala", 10) diff --git a/src/test/scala/tutor/DirectoryScannerSpec.scala b/src/test/scala/tutor/DirectoryScannerSpec.scala index bb615cf..e8323f7 100644 --- a/src/test/scala/tutor/DirectoryScannerSpec.scala +++ b/src/test/scala/tutor/DirectoryScannerSpec.scala @@ -2,13 +2,13 @@ package tutor import java.io.File -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} import tags.TestTypeTag.FunctionalTest import tutor.utils.FileUtil import scala.collection.mutable.ArrayBuffer -class DirectoryScannerSpec extends FunSpec with ShouldMatchers { +class DirectoryScannerSpec extends FunSpec with Matchers { describe("DirectoryScanner") { it("can scan directory recursively and return all file paths" + " and it should only accept known txt files", FunctionalTest) { diff --git a/src/test/scala/tutor/ReportFormatterSpec.scala b/src/test/scala/tutor/ReportFormatterSpec.scala index 8ba5b85..898b2af 100644 --- a/src/test/scala/tutor/ReportFormatterSpec.scala +++ b/src/test/scala/tutor/ReportFormatterSpec.scala @@ -1,9 +1,9 @@ package tutor -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} import tutor.utils.FileUtil -class ReportFormatterSpec extends FunSpec with ShouldMatchers { +class ReportFormatterSpec extends FunSpec with Matchers { val rf = new ReportFormatter {} diff --git a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala index 4ef6d0b..17189bf 100644 --- a/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala +++ b/src/test/scala/tutor/SourceCodeAnalyzerSpec.scala @@ -1,9 +1,9 @@ package tutor -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} import tags.TestTypeTag.FunctionalTest -class SourceCodeAnalyzerSpec extends FunSpec with ShouldMatchers { +class SourceCodeAnalyzerSpec extends FunSpec with Matchers { describe("SourceCode object") { it("can read file and create a SourceCode instance", FunctionalTest) { val sca = new SourceCodeAnalyzer {} diff --git a/src/test/scala/tutor/utils/FileUtilSpec.scala b/src/test/scala/tutor/utils/FileUtilSpec.scala index cc15046..82385d6 100644 --- a/src/test/scala/tutor/utils/FileUtilSpec.scala +++ b/src/test/scala/tutor/utils/FileUtilSpec.scala @@ -1,8 +1,8 @@ package tutor.utils -import org.scalatest.{FunSpec, ShouldMatchers} +import org.scalatest.{FunSpec, Matchers} -class FileUtilSpec extends FunSpec with ShouldMatchers { +class FileUtilSpec extends FunSpec with Matchers { describe("FileUtil"){ it("can extract file extension name"){ val path = "src/test/build.sbt" From 7f5247f80fbae63fcaa4ca61bd2b0eb42b88806d Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 19 Jun 2017 17:35:07 +0800 Subject: [PATCH 090/116] akka stream implementation --- .../tutor/CodebaseAnalyzerStreamApp.scala | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala diff --git a/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala new file mode 100644 index 0000000..b330a95 --- /dev/null +++ b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala @@ -0,0 +1,34 @@ +package tutor + +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import akka.stream.scaladsl._ +import com.typesafe.scalalogging.slf4j.StrictLogging +import tutor.utils.BenchmarkUtil + +import scala.util.{Failure, Success} + +object CodebaseAnalyzerStreamApp extends App with DirectoryScanner with SourceCodeAnalyzer with ReportFormatter with StrictLogging { + + implicit val system = ActorSystem("CodebaseAnalyzer") + implicit val materializer = ActorMaterializer() + + val path = args(0) + val beginTime = BenchmarkUtil.recordStart(s"analyze $path with akka stream") + val files = scan(path, PresetFilters.knownFileTypes, PresetFilters.ignoreFolders).iterator + val done = Source.fromIterator(() => files).map(processFile).fold(CodebaseInfo.empty) { + (acc, trySourceCodeInfo) => + trySourceCodeInfo match { + case Success(sourceCodeInfo) => acc + sourceCodeInfo + case Failure(e) => { + logger.warn("error processing file", e) + acc + } + } + }.runForeach(codebaseInfo => println(format(codebaseInfo))) + implicit val ec = system.dispatcher + done.onComplete { _ => + BenchmarkUtil.recordElapse(s"analyze $path with akka stream", beginTime) + system.terminate() + } +} From d7abceb96b718f03fc209ceff1b69e34d15c87f8 Mon Sep 17 00:00:00 2001 From: notyy Date: Mon, 19 Jun 2017 18:00:34 +0800 Subject: [PATCH 091/116] make file processing async --- .../scala/tutor/CodebaseAnalyzerStreamApp.scala | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala index b330a95..a5d9cd3 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala @@ -6,27 +6,36 @@ import akka.stream.scaladsl._ import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.utils.BenchmarkUtil +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.Future import scala.util.{Failure, Success} object CodebaseAnalyzerStreamApp extends App with DirectoryScanner with SourceCodeAnalyzer with ReportFormatter with StrictLogging { implicit val system = ActorSystem("CodebaseAnalyzer") implicit val materializer = ActorMaterializer() + implicit val ec = system.dispatcher val path = args(0) val beginTime = BenchmarkUtil.recordStart(s"analyze $path with akka stream") val files = scan(path, PresetFilters.knownFileTypes, PresetFilters.ignoreFolders).iterator - val done = Source.fromIterator(() => files).map(processFile).fold(CodebaseInfo.empty) { + var errorProcessingFiles: ArrayBuffer[Throwable] = ArrayBuffer.empty + + val done = Source.fromIterator(() => files).mapAsync(8)(path => Future { + processFile(path) + }).fold(CodebaseInfo.empty) { (acc, trySourceCodeInfo) => trySourceCodeInfo match { case Success(sourceCodeInfo) => acc + sourceCodeInfo case Failure(e) => { - logger.warn("error processing file", e) + errorProcessingFiles += e acc } } - }.runForeach(codebaseInfo => println(format(codebaseInfo))) - implicit val ec = system.dispatcher + }.runForeach(codebaseInfo => { + println(format(codebaseInfo)) + println(s"there are ${errorProcessingFiles.size} files failed to process.") + }) done.onComplete { _ => BenchmarkUtil.recordElapse(s"analyze $path with akka stream", beginTime) system.terminate() From b7d2d3d35ca3913d7238bf2a95327cd232359a3c Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 23 Jul 2017 23:29:58 +0800 Subject: [PATCH 092/116] add slick support --- build.sbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sbt b/build.sbt index 98ebc52..7488081 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,8 @@ libraryDependencies ++= Seq( "org.pegdown" % "pegdown" % "1.0.2" % "test", //used in html report "org.scalatest" %% "scalatest" % "3.0.1" % "test", "org.slf4j" % "slf4j-api" % "1.7.7", + "com.typesafe.slick" %% "slick" % "3.2.1", + "com.typesafe.slick" %% "slick-hikaricp" % "3.2.1", "ch.qos.logback" % "logback-classic" % "1.1.2", "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", "com.typesafe.akka" %% "akka-actor" % "2.5.2", From 4adcb10a73a1464ab332cbfac74069a28999693a Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 23 Jul 2017 23:30:23 +0800 Subject: [PATCH 093/116] add in memory h2 database configuration --- src/main/resources/application.conf | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/main/resources/application.conf diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf new file mode 100644 index 0000000..1601bc7 --- /dev/null +++ b/src/main/resources/application.conf @@ -0,0 +1,6 @@ +h2mem1 = { + url = "jdbc:h2:mem:test1" + driver = org.h2.Driver + connectionPool = disabled + keepAliveConnection = true +} \ No newline at end of file From 4ef36fc7cb990e5f4e878120ac5df5c8e727a193 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 25 Jul 2017 10:21:21 +0800 Subject: [PATCH 094/116] add h2database support --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 7488081..fd3f7c9 100644 --- a/build.sbt +++ b/build.sbt @@ -37,7 +37,8 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-stream-testkit" % "2.5.2", "com.typesafe.akka" %% "akka-testkit" % "2.5.2", "com.typesafe.akka" %% "akka-typed" % "2.5.2", - "com.typesafe.akka" %% "akka-contrib" % "2.5.2" + "com.typesafe.akka" %% "akka-contrib" % "2.5.2", + "com.h2database" % "h2" % "1.4.196" ) // TODO reopen it later From 980b7bdf92bbb104ee93b5fd54e9057b366732ca Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 25 Jul 2017 10:29:01 +0800 Subject: [PATCH 095/116] add a AnalyzeHistoryRecord to save analyze result to database --- src/main/scala/tutor/AnalyzeHistoryRecorder.scala | 5 +++++ src/main/scala/tutor/CodebaseAnalyzer.scala | 6 ++++-- src/main/scala/tutor/CodebaseAnalyzerParImpl.scala | 2 +- src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala | 2 +- src/main/scala/tutor/MainApp.scala | 4 ++-- src/test/scala/tutor/CodebaseAnalyzerSpec.scala | 8 ++++---- 6 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 src/main/scala/tutor/AnalyzeHistoryRecorder.scala diff --git a/src/main/scala/tutor/AnalyzeHistoryRecorder.scala b/src/main/scala/tutor/AnalyzeHistoryRecorder.scala new file mode 100644 index 0000000..013df9e --- /dev/null +++ b/src/main/scala/tutor/AnalyzeHistoryRecorder.scala @@ -0,0 +1,5 @@ +package tutor + +trait AnalyzeHistoryRecorder { + def record(codebaseInfo: CodebaseInfo):Unit = println(s"saving to database: $codebaseInfo") +} diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index 573b4dd..eefa97f 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -37,7 +37,7 @@ case class CodebaseInfo(totalFileNums: Int, fileTypeNums: Map[String, Int], tota } trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder=> override def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { val files = BenchmarkUtil.record("scan folders") { @@ -50,7 +50,9 @@ trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { processSourceFiles(files) } BenchmarkUtil.record("make last result ##") { - Some(sourceCodeInfos.foldLeft(CodebaseInfo.empty)(_ + _)) + val codebaseInfo = sourceCodeInfos.foldLeft(CodebaseInfo.empty)(_ + _) + record(codebaseInfo) + Some(codebaseInfo) } } } diff --git a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala index ff24d64..d8f68d4 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerParImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector diff --git a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala index 8c6dce1..684f4ac 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala @@ -3,7 +3,7 @@ package tutor import tutor.utils.FileUtil.Path trait CodebaseAnalyzerSeqImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer => + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.map(processFile).filter(_.isSuccess).map(_.get) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 0248adf..449f33f 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -15,10 +15,10 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val file = new File(path) val analyzer = args.find(_.startsWith("-p")).map { _ => logger.info("using par collection mode") - new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer + new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder }.getOrElse { logger.info("using sequence collection mode") - new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer + new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder } val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index 40c31cb..d03077c 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -8,7 +8,7 @@ import scala.util.{Success, Try} class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen { - val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { + val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): Try[SourceCodeInfo] = path match { @@ -26,7 +26,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen feature("analyze source code folder and give statistic results") { scenario("when directory scanner returns empty, code analyzer should return None") { Given("a directory contains no source code file") - val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer { + val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? @@ -39,7 +39,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen Given("source code folder") //use test/fixutre as test data When("analyze the folder") - val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer + val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder val analyzeResult = codeAnalyzer.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) Then("it should return correct result") analyzeResult shouldBe 'defined @@ -53,7 +53,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen codeBaseInfo.top10Files.length shouldBe 2 codeBaseInfo.totalLineCount shouldBe 31 //test par implementation - val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer + val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar } From 2e874c192f815421160dd1e015e59f1ac88f4c18 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 27 Jul 2017 23:07:42 +0800 Subject: [PATCH 096/116] save data to a h2 database --- src/main/resources/application.conf | 4 +- .../scala/tutor/AnalyzeResultHistory.scala | 8 ++++ src/main/scala/tutor/CodebaseAnalyzer.scala | 6 ++- .../scala/tutor/CodebaseAnalyzerParImpl.scala | 3 +- .../scala/tutor/CodebaseAnalyzerSeqImpl.scala | 3 +- src/main/scala/tutor/MainApp.scala | 5 ++- .../scala/tutor/repo/DBConfigProvider.scala | 28 +++++++++++++ src/main/scala/tutor/repo/Schemas.scala | 42 +++++++++++++++++++ .../scala/tutor/CodebaseAnalyzerSpec.scala | 9 ++-- .../repo/AnalyzeHistoryRepositoryTest.scala | 30 +++++++++++++ 10 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 src/main/scala/tutor/AnalyzeResultHistory.scala create mode 100644 src/main/scala/tutor/repo/DBConfigProvider.scala create mode 100644 src/main/scala/tutor/repo/Schemas.scala create mode 100644 src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 1601bc7..9df2cc1 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,5 +1,7 @@ h2mem1 = { - url = "jdbc:h2:mem:test1" + url = "jdbc:h2:retry:~/temp/h2data;DB_CLOSE_DELAY=-1" +// url = "jdbc:h2:file:./target/h2data;DB_CLOSE_DELAY=-1" +// url = "jdbc:h2:mem:test1;DB_CLOSE_DELAY=-1" driver = org.h2.Driver connectionPool = disabled keepAliveConnection = true diff --git a/src/main/scala/tutor/AnalyzeResultHistory.scala b/src/main/scala/tutor/AnalyzeResultHistory.scala new file mode 100644 index 0000000..1b7d868 --- /dev/null +++ b/src/main/scala/tutor/AnalyzeResultHistory.scala @@ -0,0 +1,8 @@ +package tutor + +//we will directly save the json representation of codebase info to database +final case class AnalyzeResultHistory(path: String, created: String, codebaseInfo: String) + + + + diff --git a/src/main/scala/tutor/CodebaseAnalyzer.scala b/src/main/scala/tutor/CodebaseAnalyzer.scala index eefa97f..2d9eaee 100644 --- a/src/main/scala/tutor/CodebaseAnalyzer.scala +++ b/src/main/scala/tutor/CodebaseAnalyzer.scala @@ -1,7 +1,9 @@ package tutor +import tutor.repo.AnalyzeHistoryRepository import tutor.utils.{BenchmarkUtil, FileUtil} import tutor.utils.FileUtil._ + import scala.math.max object CodebaseInfo { @@ -37,7 +39,7 @@ case class CodebaseInfo(totalFileNums: Int, fileTypeNums: Map[String, Int], tota } trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { - this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder=> + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository=> override def analyze(path: Path, knownFileTypes: Set[String], ignoreFolders: Set[String]): Option[CodebaseInfo] = { val files = BenchmarkUtil.record("scan folders") { @@ -51,7 +53,7 @@ trait CodebaseAnalyzer extends CodebaseAnalyzerInterface { } BenchmarkUtil.record("make last result ##") { val codebaseInfo = sourceCodeInfos.foldLeft(CodebaseInfo.empty)(_ + _) - record(codebaseInfo) + record(path, codebaseInfo) Some(codebaseInfo) } } diff --git a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala index d8f68d4..0828dcb 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerParImpl.scala @@ -1,9 +1,10 @@ package tutor +import tutor.repo.AnalyzeHistoryRepository import tutor.utils.FileUtil.Path trait CodebaseAnalyzerParImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder => + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.par.map(processFile).filter(_.isSuccess).map(_.get).toVector diff --git a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala index 684f4ac..8ef33ba 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerSeqImpl.scala @@ -1,9 +1,10 @@ package tutor +import tutor.repo.AnalyzeHistoryRepository import tutor.utils.FileUtil.Path trait CodebaseAnalyzerSeqImpl extends CodebaseAnalyzer { - this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder => + this: DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository => override protected def processSourceFiles(files: Seq[Path]): Seq[SourceCodeInfo] = { files.map(processFile).filter(_.isSuccess).map(_.get) diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index 449f33f..c2e9ac2 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -4,6 +4,7 @@ import java.io.File import com.typesafe.scalalogging.slf4j.StrictLogging import tutor.PresetFilters.{ignoreFolders, knownFileTypes} +import tutor.repo.{AnalyzeHistoryRepository, H2DB} import tutor.utils.FileUtil.Path import tutor.utils.{BenchmarkUtil, WriteSupport} @@ -15,10 +16,10 @@ object MainApp extends App with ReportFormatter with WriteSupport with StrictLog val file = new File(path) val analyzer = args.find(_.startsWith("-p")).map { _ => logger.info("using par collection mode") - new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder + new CodebaseAnalyzerParImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB }.getOrElse { logger.info("using sequence collection mode") - new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder + new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB } val rs = if (file.isFile) { analyzer.processFile(file.getAbsolutePath).map(format).getOrElse(s"error processing $path") diff --git a/src/main/scala/tutor/repo/DBConfigProvider.scala b/src/main/scala/tutor/repo/DBConfigProvider.scala new file mode 100644 index 0000000..1184598 --- /dev/null +++ b/src/main/scala/tutor/repo/DBConfigProvider.scala @@ -0,0 +1,28 @@ +package tutor.repo + +import slick.dbio.{DBIOAction, NoStream} +import slick.jdbc.{H2Profile, JdbcProfile, OracleProfile} + +import scala.concurrent.Future + +trait DBConfigProvider { + val jdbcProfile: JdbcProfile + def run[T](action: DBIOAction[T, NoStream, Nothing]):Future[T] +} + +trait OracleDB extends DBConfigProvider { + val jdbcProfile: JdbcProfile = OracleProfile +} + +trait H2DB extends DBConfigProvider { + val jdbcProfile: JdbcProfile = H2Profile + + def run[T](action: DBIOAction[T, NoStream, Nothing]):Future[T] = { + import jdbcProfile.api._ + + val db = Database.forConfig("h2mem1") + try { + db.run(action) + }finally db.close() + } +} \ No newline at end of file diff --git a/src/main/scala/tutor/repo/Schemas.scala b/src/main/scala/tutor/repo/Schemas.scala new file mode 100644 index 0000000..a9d136b --- /dev/null +++ b/src/main/scala/tutor/repo/Schemas.scala @@ -0,0 +1,42 @@ +package tutor.repo + +import tutor.AnalyzeResultHistory + +import scala.concurrent.Future + +trait Schemas { + this: DBConfigProvider => + + import jdbcProfile.api._ + + class AnalyzeResultHistoryTable(tag: Tag) extends Table[AnalyzeResultHistory](tag, "analyze_history") { + def path = column[String]("PATH") + + def created = column[String]("CREATED") + + def codeBaseInfo = column[String]("CODEBASE_INFO") + + def * = (path, created, codeBaseInfo) <> (AnalyzeResultHistory.tupled, AnalyzeResultHistory.unapply) + } + + val analyzeResultHistories = TableQuery[AnalyzeResultHistoryTable] + + def setupDB(): Future[Unit] = { + println("create tables:") + analyzeResultHistories.schema.createStatements.foreach(println) + val setUp = DBIO.seq( + analyzeResultHistories.schema.create + ) + run(setUp) + } + + def dropDB(): Future[Unit] = { + println("delete tables:") + val drop = DBIO.seq( + analyzeResultHistories.schema.drop + ) + run(drop) + } +} + +object Schemas extends Schemas with H2DB diff --git a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala index d03077c..963a002 100644 --- a/src/test/scala/tutor/CodebaseAnalyzerSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzerSpec.scala @@ -2,13 +2,14 @@ package tutor import _root_.tags.TestTypeTag.FunctionalTest import org.scalatest._ +import tutor.repo.{AnalyzeHistoryRepository, H2DB} import tutor.utils.FileUtil.Path import scala.util.{Success, Try} class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen { - val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder { + val codeBaseAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = List("a.scala", "b.scala", "c.sbt", "d") override def processFile(path: Path): Try[SourceCodeInfo] = path match { @@ -26,7 +27,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen feature("analyze source code folder and give statistic results") { scenario("when directory scanner returns empty, code analyzer should return None") { Given("a directory contains no source code file") - val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder { + val emptyCodeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB { override def scan(path: Path, knowFileTypes: Set[String], ignoreFolders: Set[String]): Seq[Path] = Vector[Path]() override def processFile(path: Path): Try[SourceCodeInfo] = ??? @@ -39,7 +40,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen Given("source code folder") //use test/fixutre as test data When("analyze the folder") - val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder + val codeAnalyzer = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB val analyzeResult = codeAnalyzer.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) Then("it should return correct result") analyzeResult shouldBe 'defined @@ -53,7 +54,7 @@ class CodebaseAnalyzerSpec extends FeatureSpec with Matchers with GivenWhenThen codeBaseInfo.top10Files.length shouldBe 2 codeBaseInfo.totalLineCount shouldBe 31 //test par implementation - val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRecorder + val codeAnalyzerParImpl = new CodebaseAnalyzerSeqImpl with DirectoryScanner with SourceCodeAnalyzer with AnalyzeHistoryRepository with H2DB val analyzeResultOfPar = codeAnalyzerParImpl.analyze("src/test/fixture", PresetFilters.knownFileTypes, PresetFilters.ignoreFolders) analyzeResult shouldBe analyzeResultOfPar } diff --git a/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala b/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala new file mode 100644 index 0000000..af145b9 --- /dev/null +++ b/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala @@ -0,0 +1,30 @@ +package tutor.repo + +import scala.concurrent.ExecutionContext.Implicits.global +import org.scalatest.{BeforeAndAfter, FunSpec, Matchers} +import tutor.CodebaseInfo + +import scala.concurrent.Await +import scala.concurrent.duration._ + +class AnalyzeHistoryRepositoryTest extends FunSpec with Matchers with Schemas with H2DB + with AnalyzeHistoryRepository with BeforeAndAfter { + + before { + Await.result(dropDB(), 5 seconds) + } + + describe("AnalyzeHistoryRecorder"){ +// it("should create tables in h2"){ +// AnalyzeHistoryRecorder.setupDB() +// } + it("can insert analyzeHistory"){ + val c = Await.result(setupDB() + .flatMap{ _ => + record("some path",CodebaseInfo(1, Map("java" -> 1), 1, 10,None,Nil)) + } + , 10 seconds) + c shouldBe 1 + } + } +} From 906db7c774ba8c178398518bcbeb1bf4a7324091 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 27 Jul 2017 23:07:54 +0800 Subject: [PATCH 097/116] save data to a h2 database --- .../scala/tutor/AnalyzeHistoryRecorder.scala | 5 ---- .../tutor/repo/AnalyzeHistoryRepository.scala | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) delete mode 100644 src/main/scala/tutor/AnalyzeHistoryRecorder.scala create mode 100644 src/main/scala/tutor/repo/AnalyzeHistoryRepository.scala diff --git a/src/main/scala/tutor/AnalyzeHistoryRecorder.scala b/src/main/scala/tutor/AnalyzeHistoryRecorder.scala deleted file mode 100644 index 013df9e..0000000 --- a/src/main/scala/tutor/AnalyzeHistoryRecorder.scala +++ /dev/null @@ -1,5 +0,0 @@ -package tutor - -trait AnalyzeHistoryRecorder { - def record(codebaseInfo: CodebaseInfo):Unit = println(s"saving to database: $codebaseInfo") -} diff --git a/src/main/scala/tutor/repo/AnalyzeHistoryRepository.scala b/src/main/scala/tutor/repo/AnalyzeHistoryRepository.scala new file mode 100644 index 0000000..5f56ffd --- /dev/null +++ b/src/main/scala/tutor/repo/AnalyzeHistoryRepository.scala @@ -0,0 +1,25 @@ +package tutor.repo + +import java.text.SimpleDateFormat +import java.util.Date + +import tutor.{AnalyzeResultHistory, CodebaseInfo} + +import scala.concurrent.Future + + +trait AnalyzeHistoryRepository { + this: DBConfigProvider => + + import Schemas._ + import jdbcProfile.api._ + + + def record(path: String, codebaseInfo: CodebaseInfo): Future[Int] = { + val created = new SimpleDateFormat("yyyyMMdd").format(new Date()) + val analyzeResultHistory = AnalyzeResultHistory(path, created, codebaseInfo.toString) + + val q = analyzeResultHistories += analyzeResultHistory + run(q) + } +} \ No newline at end of file From 100bf7fd504ee0ae2dda059542d2fb16dcc60870 Mon Sep 17 00:00:00 2001 From: notyy Date: Thu, 27 Jul 2017 23:08:41 +0800 Subject: [PATCH 098/116] use h2:retry to solve thread interrupt problem --- src/main/resources/application.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 9df2cc1..1aa40e9 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,4 +1,5 @@ h2mem1 = { + // retry is a strange solution to solve thread interrupt problem url = "jdbc:h2:retry:~/temp/h2data;DB_CLOSE_DELAY=-1" // url = "jdbc:h2:file:./target/h2data;DB_CLOSE_DELAY=-1" // url = "jdbc:h2:mem:test1;DB_CLOSE_DELAY=-1" From 348e2cc72295add1847346b20efd33102f5b7cb9 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 10:55:07 +0800 Subject: [PATCH 099/116] upgrade scala version and some other libs --- build.sbt | 48 +++++++++---------- project/plugins.sbt | 2 +- .../tutor/CodebaseAnalyzerStreamApp.scala | 2 +- src/main/scala/tutor/DirectoryScanner.scala | 2 +- src/main/scala/tutor/MainApp.scala | 2 +- src/main/scala/tutor/SourceCodeInfo.scala | 4 +- .../scala/tutor/utils/BenchmarkUtil.scala | 2 +- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/build.sbt b/build.sbt index fd3f7c9..50abd69 100644 --- a/build.sbt +++ b/build.sbt @@ -7,37 +7,37 @@ isSnapshot := true organization := "com.github.notyy" // set the Scala version used for the project -scalaVersion := "2.11.8" +scalaVersion := "2.12.2" libraryDependencies ++= Seq( - "org.scalacheck" %% "scalacheck" % "1.13.2" % "test", - "org.pegdown" % "pegdown" % "1.0.2" % "test", //used in html report + "org.scalacheck" %% "scalacheck" % "1.13.4" % "test", + "org.pegdown" % "pegdown" % "1.6.0" % "test", //used in html report "org.scalatest" %% "scalatest" % "3.0.1" % "test", "org.slf4j" % "slf4j-api" % "1.7.7", "com.typesafe.slick" %% "slick" % "3.2.1", "com.typesafe.slick" %% "slick-hikaricp" % "3.2.1", "ch.qos.logback" % "logback-classic" % "1.1.2", - "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", - "com.typesafe.akka" %% "akka-actor" % "2.5.2", - "com.typesafe.akka" %% "akka-agent" % "2.5.2", - "com.typesafe.akka" %% "akka-camel" % "2.5.2", - "com.typesafe.akka" %% "akka-cluster" % "2.5.2", - "com.typesafe.akka" %% "akka-cluster-metrics" % "2.5.2", - "com.typesafe.akka" %% "akka-cluster-sharding" % "2.5.2", - "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.2", - "com.typesafe.akka" %% "akka-distributed-data" % "2.5.2", - "com.typesafe.akka" %% "akka-multi-node-testkit" % "2.5.2", - "com.typesafe.akka" %% "akka-osgi" % "2.5.2", - "com.typesafe.akka" %% "akka-persistence" % "2.5.2", - "com.typesafe.akka" %% "akka-persistence-query" % "2.5.2", - "com.typesafe.akka" %% "akka-persistence-tck" % "2.5.2", - "com.typesafe.akka" %% "akka-remote" % "2.5.2", - "com.typesafe.akka" %% "akka-slf4j" % "2.5.2", - "com.typesafe.akka" %% "akka-stream" % "2.5.2", - "com.typesafe.akka" %% "akka-stream-testkit" % "2.5.2", - "com.typesafe.akka" %% "akka-testkit" % "2.5.2", - "com.typesafe.akka" %% "akka-typed" % "2.5.2", - "com.typesafe.akka" %% "akka-contrib" % "2.5.2", + "com.typesafe.scala-logging" %% "scala-logging" % "3.7.1", + "com.typesafe.akka" %% "akka-actor" % "2.5.3", + "com.typesafe.akka" %% "akka-agent" % "2.5.3", + "com.typesafe.akka" %% "akka-camel" % "2.5.3", + "com.typesafe.akka" %% "akka-cluster" % "2.5.3", + "com.typesafe.akka" %% "akka-cluster-metrics" % "2.5.3", + "com.typesafe.akka" %% "akka-cluster-sharding" % "2.5.3", + "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.3", + "com.typesafe.akka" %% "akka-distributed-data" % "2.5.3", + "com.typesafe.akka" %% "akka-multi-node-testkit" % "2.5.3", + "com.typesafe.akka" %% "akka-osgi" % "2.5.3", + "com.typesafe.akka" %% "akka-persistence" % "2.5.3", + "com.typesafe.akka" %% "akka-persistence-query" % "2.5.3", + "com.typesafe.akka" %% "akka-persistence-tck" % "2.5.3", + "com.typesafe.akka" %% "akka-remote" % "2.5.3", + "com.typesafe.akka" %% "akka-slf4j" % "2.5.3", + "com.typesafe.akka" %% "akka-stream" % "2.5.3", + "com.typesafe.akka" %% "akka-stream-testkit" % "2.5.3", + "com.typesafe.akka" %% "akka-testkit" % "2.5.3", + "com.typesafe.akka" %% "akka-typed" % "2.5.3", + "com.typesafe.akka" %% "akka-contrib" % "2.5.3", "com.h2database" % "h2" % "1.4.196" ) diff --git a/project/plugins.sbt b/project/plugins.sbt index f38daac..a6b959d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") diff --git a/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala index a5d9cd3..6e75c75 100644 --- a/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala +++ b/src/main/scala/tutor/CodebaseAnalyzerStreamApp.scala @@ -3,7 +3,7 @@ package tutor import akka.actor.ActorSystem import akka.stream.ActorMaterializer import akka.stream.scaladsl._ -import com.typesafe.scalalogging.slf4j.StrictLogging +import com.typesafe.scalalogging.StrictLogging import tutor.utils.BenchmarkUtil import scala.collection.mutable.ArrayBuffer diff --git a/src/main/scala/tutor/DirectoryScanner.scala b/src/main/scala/tutor/DirectoryScanner.scala index dec0975..45a47db 100644 --- a/src/main/scala/tutor/DirectoryScanner.scala +++ b/src/main/scala/tutor/DirectoryScanner.scala @@ -2,7 +2,7 @@ package tutor import java.io.File -import com.typesafe.scalalogging.slf4j.StrictLogging +import com.typesafe.scalalogging.StrictLogging import tutor.utils.FileUtil import tutor.utils.FileUtil.Path diff --git a/src/main/scala/tutor/MainApp.scala b/src/main/scala/tutor/MainApp.scala index c2e9ac2..683c3dc 100644 --- a/src/main/scala/tutor/MainApp.scala +++ b/src/main/scala/tutor/MainApp.scala @@ -2,7 +2,7 @@ package tutor import java.io.File -import com.typesafe.scalalogging.slf4j.StrictLogging +import com.typesafe.scalalogging.StrictLogging import tutor.PresetFilters.{ignoreFolders, knownFileTypes} import tutor.repo.{AnalyzeHistoryRepository, H2DB} import tutor.utils.FileUtil.Path diff --git a/src/main/scala/tutor/SourceCodeInfo.scala b/src/main/scala/tutor/SourceCodeInfo.scala index bdaf748..099415c 100644 --- a/src/main/scala/tutor/SourceCodeInfo.scala +++ b/src/main/scala/tutor/SourceCodeInfo.scala @@ -1,8 +1,8 @@ package tutor -import com.typesafe.scalalogging.slf4j.StrictLogging -import tutor.utils.FileUtil._ +import com.typesafe.scalalogging.StrictLogging import tutor.utils.FileUtil +import tutor.utils.FileUtil._ import scala.util.Try diff --git a/src/main/scala/tutor/utils/BenchmarkUtil.scala b/src/main/scala/tutor/utils/BenchmarkUtil.scala index 40542b0..081679c 100644 --- a/src/main/scala/tutor/utils/BenchmarkUtil.scala +++ b/src/main/scala/tutor/utils/BenchmarkUtil.scala @@ -3,7 +3,7 @@ package tutor.utils import java.text.SimpleDateFormat import java.util.Date -import com.typesafe.scalalogging.slf4j.StrictLogging +import com.typesafe.scalalogging.StrictLogging object BenchmarkUtil extends StrictLogging { def record[T](actionDesc: String)(action: => T): T = { From 88aba5688e33715a97016b45504195633ade48bc Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:25:17 +0800 Subject: [PATCH 100/116] add logback config --- src/main/resources/logback.xml | 22 ++++++++++++++++++++++ src/test/resources/logback-test.xml | 26 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 src/main/resources/logback.xml create mode 100644 src/test/resources/logback-test.xml diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..8813133 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,22 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + logs/sbtTemplate.log + + %d{HH:mm:ss} %level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..9072c2e --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,26 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + logs/sbtTemplate.log + + %d{HH:mm:ss} %level %logger{36} - %msg%n + + + + + + + + + + + + + + + \ No newline at end of file From 2c98ac7db4515411e5178a9835c7ed6c2230ab6d Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:25:34 +0800 Subject: [PATCH 101/116] use memory db instead of file --- src/main/resources/application.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 1aa40e9..84241e9 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,8 +1,8 @@ h2mem1 = { // retry is a strange solution to solve thread interrupt problem - url = "jdbc:h2:retry:~/temp/h2data;DB_CLOSE_DELAY=-1" +// url = "jdbc:h2:retry:~/temp/h2data;DB_CLOSE_DELAY=-1" // url = "jdbc:h2:file:./target/h2data;DB_CLOSE_DELAY=-1" -// url = "jdbc:h2:mem:test1;DB_CLOSE_DELAY=-1" + url = "jdbc:h2:mem:test1;DB_CLOSE_DELAY=-1" driver = org.h2.Driver connectionPool = disabled keepAliveConnection = true From a3469c3157f4468a8831e841ca6394fc137db547 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:26:15 +0800 Subject: [PATCH 102/116] should setup db before drop it --- .../scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala b/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala index af145b9..1e97f4c 100644 --- a/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala +++ b/src/test/scala/tutor/repo/AnalyzeHistoryRepositoryTest.scala @@ -11,6 +11,10 @@ class AnalyzeHistoryRepositoryTest extends FunSpec with Matchers with Schemas wi with AnalyzeHistoryRepository with BeforeAndAfter { before { + Await.result(setupDB(), 5 seconds) + } + + after { Await.result(dropDB(), 5 seconds) } @@ -19,10 +23,8 @@ class AnalyzeHistoryRepositoryTest extends FunSpec with Matchers with Schemas wi // AnalyzeHistoryRecorder.setupDB() // } it("can insert analyzeHistory"){ - val c = Await.result(setupDB() - .flatMap{ _ => + val c = Await.result( record("some path",CodebaseInfo(1, Map("java" -> 1), 1, 10,None,Nil)) - } , 10 seconds) c shouldBe 1 } From d3b5651c577b47aa3243a20b671abcf6674f94e0 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:26:40 +0800 Subject: [PATCH 103/116] should not use local file path in test --- src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala index 4329025..7806e87 100644 --- a/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala +++ b/src/test/scala/tutor/CodebaseAnalyzeAggregatorActorSpec.scala @@ -23,7 +23,7 @@ class CodebaseAnalyzeAggregatorActorSpec extends FunSpec with Matchers { result.avgLineCount shouldBe 15.0 result.longestFileInfo.get.localPath shouldBe "SomeCode.scala" result.top10Files should have size 2 - result.top10Files should contain (SourceCodeInfo("/Users/twer/source/scala/CodeAnalyzerTutorial/src/test/fixture/sub/SomeCode.scala", "SomeCode.scala", 16)) + result.top10Files.map(file => (file.localPath,file.lineCount)) should contain (("SomeCode.scala", 16)) } } } From 58b104b4c14e4d943564211f1cf2dbfbbb96de1b Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:32:15 +0800 Subject: [PATCH 104/116] update Jenkinsfile, because after upgrade scala to 2.12, the output file becomes scala-2.12, so everything should change accordingly. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7ceebe2..279b689 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { steps { sh 'sbt coverage test' sh 'sbt coverageReport' - sh 'cp -R target/scala-2.11/scoverage-report ./report/' + sh 'cp -R target/scala-2.12/scoverage-report ./report/' } } stage('rebuild without coverage') { @@ -45,7 +45,7 @@ pipeline { } stage('deploy') { steps { - sh 'cp target/scala-2.11/CodeAnalyzerTutorial-assembly-0.0.1.jar /Users/twer/dev/bin/CodeAnalyzer.jar' + sh 'cp target/scala-2.12/CodeAnalyzerTutorial-assembly-0.0.1.jar /Users/twer/dev/bin/CodeAnalyzer.jar' } } stage('health check') { From af64e5409987d6ee0de2012fd55196d82bbaa115 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 15:43:49 +0800 Subject: [PATCH 105/116] there are conflict about osgi when using sbt assembly, so I remove akka-osgi since I haven't use it. --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 50abd69..500c144 100644 --- a/build.sbt +++ b/build.sbt @@ -27,7 +27,6 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.3", "com.typesafe.akka" %% "akka-distributed-data" % "2.5.3", "com.typesafe.akka" %% "akka-multi-node-testkit" % "2.5.3", - "com.typesafe.akka" %% "akka-osgi" % "2.5.3", "com.typesafe.akka" %% "akka-persistence" % "2.5.3", "com.typesafe.akka" %% "akka-persistence-query" % "2.5.3", "com.typesafe.akka" %% "akka-persistence-tck" % "2.5.3", From abe9365cfec1751960ab6b818d1a3a59f2fdc2ca Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 16:58:40 +0800 Subject: [PATCH 106/116] test Jenkins file --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 279b689..1563546 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,4 @@ +Jenkinsfile (Declarative Pipeline) pipeline { agent any stages { From 5706e10781678bd3a28c20fd7079ea5294469145 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 16:59:41 +0800 Subject: [PATCH 107/116] remove the wrong header of Jenkins file --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1563546..279b689 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,3 @@ -Jenkinsfile (Declarative Pipeline) pipeline { agent any stages { From 4b8811fb114cb975bfc9f9c503c32636d08aec1c Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 17:09:14 +0800 Subject: [PATCH 108/116] add a human confirm stage before deploy --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 279b689..479a23a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,9 +38,9 @@ pipeline { sh 'sbt assembly' } } - stage('predeploy test') { + stage('Sanity check') { steps { - sh 'echo predeploy test' + input "confirm to deploy?" } } stage('deploy') { From 08b074cfef10c56138a1b44cfe37524e7d48f5f6 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 22:59:58 +0800 Subject: [PATCH 109/116] remove useless deploy steps --- Jenkinsfile | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 479a23a..13e7837 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -38,20 +38,5 @@ pipeline { sh 'sbt assembly' } } - stage('Sanity check') { - steps { - input "confirm to deploy?" - } - } - stage('deploy') { - steps { - sh 'cp target/scala-2.12/CodeAnalyzerTutorial-assembly-0.0.1.jar /Users/twer/dev/bin/CodeAnalyzer.jar' - } - } - stage('health check') { - steps { - sh 'echo health check' - } - } } } \ No newline at end of file From 72b0a376729e3058a5efcba65bd3a0b80be07103 Mon Sep 17 00:00:00 2001 From: notyy Date: Tue, 8 Aug 2017 23:48:28 +0800 Subject: [PATCH 110/116] echo some message --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 13e7837..cfcb45e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,6 +36,7 @@ pipeline { stage('assembly') { steps { sh 'sbt assembly' + sh 'echo assembly successfully' } } } From 1494c58bd3b3a0be047a4faaa5f855b7f5efdae0 Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 9 Aug 2017 10:15:32 +0800 Subject: [PATCH 111/116] try accessing build number --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index cfcb45e..8ea8f6a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ pipeline { stages { stage('compile') { steps { + sh 'echo build number is $buildNumber' sh 'sbt clean compile' } } From 619419e8560c49ac369ca82a712f969019bd25c4 Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 9 Aug 2017 10:32:05 +0800 Subject: [PATCH 112/116] try accessing build number --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8ea8f6a..68757a1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('compile') { steps { - sh 'echo build number is $buildNumber' + sh 'echo build number is ${env.BUILD_NUMBER}' sh 'sbt clean compile' } } From 2e7f64d053031180f293903b3e140deefa507cf5 Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 9 Aug 2017 10:39:18 +0800 Subject: [PATCH 113/116] try accessing build number --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 68757a1..df28b49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('compile') { steps { - sh 'echo build number is ${env.BUILD_NUMBER}' + sh 'echo build number is ${BUILD_NUMBER}' sh 'sbt clean compile' } } From 8679fa37c5b2e7ab82b066cfc2ba258f7ef1e878 Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 9 Aug 2017 11:40:55 +0800 Subject: [PATCH 114/116] try connect from other build --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index df28b49..7353394 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ pipeline { stages { stage('compile') { steps { + sh 'try connect from prev build' sh 'echo build number is ${BUILD_NUMBER}' sh 'sbt clean compile' } From 8a795e41af3c2d4152253bc129410a2cdf43013f Mon Sep 17 00:00:00 2001 From: notyy Date: Wed, 9 Aug 2017 11:43:03 +0800 Subject: [PATCH 115/116] try connect from other build --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7353394..3c13a0a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { stages { stage('compile') { steps { - sh 'try connect from prev build' + sh 'echo try connect from prev build' sh 'echo build number is ${BUILD_NUMBER}' sh 'sbt clean compile' } From 648ac212e264e7c23c95935d9e32110b3ce15940 Mon Sep 17 00:00:00 2001 From: notyy Date: Sun, 13 Aug 2017 22:19:21 +0800 Subject: [PATCH 116/116] add route termination logic --- src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala index e087786..eabb4a6 100644 --- a/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala +++ b/src/main/scala/tutor/CodebaseAnalyzeAggregatorActor.scala @@ -2,7 +2,7 @@ package tutor import java.util.Date -import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props} +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props, Terminated} import akka.routing.{ActorRefRoutee, RoundRobinRoutingLogic, Router} import tutor.CodebaseAnalyzeAggregatorActor.{AnalyzeDirectory, Complete, Report, Timeout} import tutor.SourceCodeAnalyzerActor.NewFile @@ -71,6 +71,11 @@ class CodebaseAnalyzeAggregatorActor extends Actor with ActorLogging with Direct controller ! Report(result) BenchmarkUtil.recordElapse(s"analyze folder $currentPath", beginTime) } + case Terminated(a) => + router = router.removeRoutee(a) + val r = context.actorOf(Props[SourceCodeAnalyzerActor]) + context watch r + router = router.addRoutee(r) case x@_ => log.error(s"receive unknown message $x") }