From 394f9418c5f972533512ae474a06bb65eed0e168 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 17 Apr 2019 13:40:21 +0300 Subject: [PATCH 1/6] Created functions looking for "tests" in paths --- core/src/main/scala/codesearch/core/search/HaskellSearch.scala | 2 ++ .../main/scala/codesearch/core/search/JavaScriptSearch.scala | 1 + core/src/main/scala/codesearch/core/search/RubySearch.scala | 1 + core/src/main/scala/codesearch/core/search/RustSearch.scala | 1 + core/src/main/scala/codesearch/core/search/Search.scala | 1 + 5 files changed, 6 insertions(+) diff --git a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala index f873d7e..68f4b5c 100644 --- a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala +++ b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala @@ -10,4 +10,6 @@ class HaskellSearch extends Search { override protected def extensions: Extensions = HaskellExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://hackage.haskell.org/package/$packageName-$version" + def ifTestInPath(path: String): Boolean = + path.matches(".*/((?i)(test|tests|testsuite|testsuites|test-suite|test-suites))/.*") } diff --git a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala index 9b6b77b..aa2f9c5 100644 --- a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala +++ b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala @@ -10,4 +10,5 @@ class JavaScriptSearch extends Search { override protected def extensions: Extensions = JavaScriptExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://www.npmjs.com/package/$packageName/v/$version" + def ifTestInPath(path: String): Boolean = path.matches(".*((?i)(test|spec|tests)).*") } diff --git a/core/src/main/scala/codesearch/core/search/RubySearch.scala b/core/src/main/scala/codesearch/core/search/RubySearch.scala index 1ac8bcf..63f1ef3 100644 --- a/core/src/main/scala/codesearch/core/search/RubySearch.scala +++ b/core/src/main/scala/codesearch/core/search/RubySearch.scala @@ -10,4 +10,5 @@ class RubySearch extends Search { override protected def extensions: Extensions = RubyExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://rubygems.org/gems/$packageName/versions/$version" + def ifTestInPath(path: String): Boolean = path.contains("/spec/") || path.contains("/test/") } diff --git a/core/src/main/scala/codesearch/core/search/RustSearch.scala b/core/src/main/scala/codesearch/core/search/RustSearch.scala index 99b3ad6..fcc7983 100644 --- a/core/src/main/scala/codesearch/core/search/RustSearch.scala +++ b/core/src/main/scala/codesearch/core/search/RustSearch.scala @@ -10,4 +10,5 @@ class RustSearch extends Search { override protected def extensions: Extensions = RustExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://docs.rs/crate/$packageName/$version" + def ifTestInPath(path: String): Boolean = path.contains("/test/") } diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index 88eb606..4039800 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -24,6 +24,7 @@ trait Search { protected def cindexDir: СindexDirectory protected def extensions: Extensions protected val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.unsafeCreate[IO] + def ifTestInPath(path: String): Boolean def search(request: SearchRequest): IO[CSearchPage] = { for { From b9393141d82452e993af21451c389a985be25b93 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 17 Apr 2019 16:35:33 +0300 Subject: [PATCH 2/6] Added tests --- .../core/search/HaskellSearch.scala | 2 +- .../core/search/JavaScriptSearch.scala | 2 +- .../codesearch/core/search/RubySearch.scala | 2 +- .../codesearch/core/search/RustSearch.scala | 2 +- .../scala/codesearch/core/search/Search.scala | 12 +++++-- .../core/search/SearchRequest.scala | 6 +++- .../core/search/HaskellSearchSpec.scala | 10 ++++++ .../core/search/JavaScriptSearchSpec.scala | 10 ++++++ .../core/search/RubySearchSpec.scala | 10 ++++++ .../core/search/RustSearchSpec.scala | 10 ++++++ .../core/search/SearchSpecBase.scala | 35 +++++++++++++++++++ 11 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala create mode 100644 core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala create mode 100644 core/src/test/scala/codesearch/core/search/RubySearchSpec.scala create mode 100644 core/src/test/scala/codesearch/core/search/RustSearchSpec.scala create mode 100644 core/src/test/scala/codesearch/core/search/SearchSpecBase.scala diff --git a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala index 68f4b5c..86d7b6a 100644 --- a/core/src/main/scala/codesearch/core/search/HaskellSearch.scala +++ b/core/src/main/scala/codesearch/core/search/HaskellSearch.scala @@ -10,6 +10,6 @@ class HaskellSearch extends Search { override protected def extensions: Extensions = HaskellExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://hackage.haskell.org/package/$packageName-$version" - def ifTestInPath(path: String): Boolean = + def isTestInPath(path: String): Boolean = path.matches(".*/((?i)(test|tests|testsuite|testsuites|test-suite|test-suites))/.*") } diff --git a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala index aa2f9c5..50f5d8e 100644 --- a/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala +++ b/core/src/main/scala/codesearch/core/search/JavaScriptSearch.scala @@ -10,5 +10,5 @@ class JavaScriptSearch extends Search { override protected def extensions: Extensions = JavaScriptExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://www.npmjs.com/package/$packageName/v/$version" - def ifTestInPath(path: String): Boolean = path.matches(".*((?i)(test|spec|tests)).*") + def isTestInPath(path: String): Boolean = path.matches(".*((?i)(test|spec|tests)).*") } diff --git a/core/src/main/scala/codesearch/core/search/RubySearch.scala b/core/src/main/scala/codesearch/core/search/RubySearch.scala index 63f1ef3..14f8d81 100644 --- a/core/src/main/scala/codesearch/core/search/RubySearch.scala +++ b/core/src/main/scala/codesearch/core/search/RubySearch.scala @@ -10,5 +10,5 @@ class RubySearch extends Search { override protected def extensions: Extensions = RubyExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://rubygems.org/gems/$packageName/versions/$version" - def ifTestInPath(path: String): Boolean = path.contains("/spec/") || path.contains("/test/") + def isTestInPath(path: String): Boolean = path.contains("/spec/") || path.contains("/test/") } diff --git a/core/src/main/scala/codesearch/core/search/RustSearch.scala b/core/src/main/scala/codesearch/core/search/RustSearch.scala index fcc7983..81e8b03 100644 --- a/core/src/main/scala/codesearch/core/search/RustSearch.scala +++ b/core/src/main/scala/codesearch/core/search/RustSearch.scala @@ -10,5 +10,5 @@ class RustSearch extends Search { override protected def extensions: Extensions = RustExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://docs.rs/crate/$packageName/$version" - def ifTestInPath(path: String): Boolean = path.contains("/test/") + def isTestInPath(path: String): Boolean = path.contains("/test/") } diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index 4039800..a87a20c 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -24,14 +24,22 @@ trait Search { protected def cindexDir: СindexDirectory protected def extensions: Extensions protected val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.unsafeCreate[IO] - def ifTestInPath(path: String): Boolean + def isTestInPath(path: String): Boolean def search(request: SearchRequest): IO[CSearchPage] = { for { lines <- csearch(request) - results <- Stream + + shippetsInfo = Stream .emits(lines) .through(SnippetsGrouper.groupLines(snippetConfig)) + + filteredShippetsInfo = if (request.withoutTests) + shippetsInfo.filter(shippetInfo => !isTestInPath(shippetInfo.filePath)) + else + shippetsInfo + + results <- filteredShippetsInfo .drop(snippetConfig.pageSize * (request.page - 1)) .take(snippetConfig.pageSize) .evalMap(createSnippet) diff --git a/core/src/main/scala/codesearch/core/search/SearchRequest.scala b/core/src/main/scala/codesearch/core/search/SearchRequest.scala index 82e3025..401f9ab 100644 --- a/core/src/main/scala/codesearch/core/search/SearchRequest.scala +++ b/core/src/main/scala/codesearch/core/search/SearchRequest.scala @@ -10,6 +10,7 @@ import com.softwaremill.sttp._ * @param spaceInsensitive space insensitive search flag * @param preciseMatch precise match flag * @param sourcesOnly sources only flag + * @param withoutTests search without tests * @param page next pagination */ case class SearchRequest( @@ -21,6 +22,7 @@ case class SearchRequest( spaceInsensitive: Boolean, preciseMatch: Boolean, sourcesOnly: Boolean, + withoutTests: Boolean, page: Int ) { @@ -32,7 +34,7 @@ case class SearchRequest( def stringify(x: Boolean): String = if (x) "on" else "off" uri"$host/$lang/search?query=$query&filter=$filter&filePath=$filePath&insensitive=${stringify(insensitive)}&space=${stringify( - spaceInsensitive)}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}" + spaceInsensitive)}&precise=${stringify(preciseMatch)}&sources=${stringify(sourcesOnly)}&withoutTests=${stringify(withoutTests)}" } } @@ -46,6 +48,7 @@ object SearchRequest { spaceInsensitive: String, preciseMatch: String, sourcesOnly: String, + withoutTests: String, page: String ): SearchRequest = { SearchRequest( @@ -57,6 +60,7 @@ object SearchRequest { isEnabled(spaceInsensitive), isEnabled(preciseMatch), isEnabled(sourcesOnly), + isEnabled(withoutTests), page.toInt, ) } diff --git a/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala b/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala new file mode 100644 index 0000000..f1aae5d --- /dev/null +++ b/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala @@ -0,0 +1,10 @@ +package codesearch.core.search + +class HaskellSearchSpec extends SearchSpecBase { + "Checking on the way to tests" - { + haskellIsTestInWay("path/to/package/test/", true) + haskellIsTestInWay("path/to/package/testSUites/", true) + haskellIsTestInWay("path/to/package/tEsts/", true) + haskellIsTestInWay("path/to/package/tes-t/", false) + } +} diff --git a/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala b/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala new file mode 100644 index 0000000..529b792 --- /dev/null +++ b/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala @@ -0,0 +1,10 @@ +package codesearch.core.search + +class JavaScriptSearchSpec extends SearchSpecBase { + "Checking on the way to tests" - { + javaScriptIsTestInWay("path/to/package/tests/", true) + javaScriptIsTestInWay("path/to/package/sPEc-class/", true) + javaScriptIsTestInWay("path/to/package/tEsts.js", true) + javaScriptIsTestInWay("path/to/package/tes-t/", false) + } +} diff --git a/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala b/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala new file mode 100644 index 0000000..a6ced1c --- /dev/null +++ b/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala @@ -0,0 +1,10 @@ +package codesearch.core.search + +class RubySearchSpec extends SearchSpecBase { + "Checking on the way to tests" - { + rubyIsTestInWay("path/to/package/test/", true) + rubyIsTestInWay("path/to/package/spec/", true) + rubyIsTestInWay("path/to/package/tEsts/", false) + rubyIsTestInWay("path/to/package/tes-t/", false) + } +} diff --git a/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala b/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala new file mode 100644 index 0000000..0ab34fc --- /dev/null +++ b/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala @@ -0,0 +1,10 @@ +package codesearch.core.search + +class RustSearchSpec extends SearchSpecBase { + "Checking on the way to tests" - { + rustIsTestInWay("path/to/package/test/", true) + rustIsTestInWay("path/to/package/spec/", false) + rustIsTestInWay("path/to/package/tEsts/", false) + rustIsTestInWay("path/to/package/tes-t/", false) + } +} diff --git a/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala b/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala new file mode 100644 index 0000000..66f6372 --- /dev/null +++ b/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala @@ -0,0 +1,35 @@ +package codesearch.core.search + +import org.scalatest.{FreeSpec, Matchers} + + +trait SearchSpecBase extends FreeSpec with Matchers{ + val haskellSearch = new HaskellSearch + val rubySearch = new RubySearch + val rustSearch = new RustSearch + val javaScriptSearch = new JavaScriptSearch + + def haskellIsTestInWay(path: String, result: Boolean): Unit = { + path in { + haskellSearch.isTestInPath(path) shouldBe result + } + } + + def rubyIsTestInWay(path: String, result: Boolean): Unit = { + path in { + rubySearch.isTestInPath(path) shouldBe result + } + } + + def rustIsTestInWay(path: String, result: Boolean): Unit = { + path in { + rustSearch.isTestInPath(path) shouldBe result + } + } + + def javaScriptIsTestInWay(path: String, result: Boolean): Unit = { + path in { + javaScriptSearch.isTestInPath(path) shouldBe result + } + } +} From 7f3a0418465992b54f278aaebf8dbe19c5b7d2f0 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 24 Apr 2019 14:13:17 +0300 Subject: [PATCH 3/6] Added checkbox for exclude tests --- .../codesearch/core/search/RustSearch.scala | 2 +- .../scala/codesearch/core/search/Search.scala | 15 ++++++++------- .../scala/codesearch/core/syntax/stream.scala | 8 +++++--- .../web/controllers/SearchController.scala | 16 ++++++++++++++-- web-server/app/views/docFrame.scala.html | 5 +++++ web-server/app/views/search.scala.html | 2 +- web-server/app/views/searchBox.scala.html | 7 ++++++- web-server/app/views/searchResults.scala.html | 3 ++- web-server/conf/routes | 8 ++++---- 9 files changed, 46 insertions(+), 20 deletions(-) diff --git a/core/src/main/scala/codesearch/core/search/RustSearch.scala b/core/src/main/scala/codesearch/core/search/RustSearch.scala index 81e8b03..3044648 100644 --- a/core/src/main/scala/codesearch/core/search/RustSearch.scala +++ b/core/src/main/scala/codesearch/core/search/RustSearch.scala @@ -10,5 +10,5 @@ class RustSearch extends Search { override protected def extensions: Extensions = RustExtensions override protected def buildRepUrl(packageName: String, version: String): String = s"https://docs.rs/crate/$packageName/$version" - def isTestInPath(path: String): Boolean = path.contains("/test/") + def isTestInPath(path: String): Boolean = path.contains("/tests/") } diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index a87a20c..3b93810 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -12,10 +12,11 @@ import codesearch.core.index.repository.Extensions import codesearch.core.search.Search.{CSearchPage, CSearchResult, CodeSnippet, Package, PackageResult, snippetConfig} import codesearch.core.search.SnippetsGrouper.SnippetInfo import codesearch.core.util.Helper.readFileAsync -import fs2.{Pipe, Stream} +import fs2.{Pipe, Pure, Stream} import io.chrisdavenport.log4cats.SelfAwareStructuredLogger import io.chrisdavenport.log4cats.slf4j.Slf4jLogger import codesearch.core.regex.RegexConstructor +import codesearch.core.syntax.stream._ import scala.sys.process.Process @@ -30,23 +31,23 @@ trait Search { for { lines <- csearch(request) - shippetsInfo = Stream + snippetsInfo = Stream .emits(lines) .through(SnippetsGrouper.groupLines(snippetConfig)) - filteredShippetsInfo = if (request.withoutTests) - shippetsInfo.filter(shippetInfo => !isTestInPath(shippetInfo.filePath)) + filteredSnippetsInfo = if (request.withoutTests) + snippetsInfo.filterNot(snippetInfo => isTestInPath(snippetInfo.filePath)) else - shippetsInfo + snippetsInfo - results <- filteredShippetsInfo + results <- filteredSnippetsInfo .drop(snippetConfig.pageSize * (request.page - 1)) .take(snippetConfig.pageSize) .evalMap(createSnippet) .through(groupByPackage) .compile .toList - } yield CSearchPage(results.sortBy(_.pack.name), lines.size) + } yield CSearchPage(results.sortBy(_.pack.name), filteredSnippetsInfo.compile.toList.size) } /** diff --git a/core/src/main/scala/codesearch/core/syntax/stream.scala b/core/src/main/scala/codesearch/core/syntax/stream.scala index 7e9377e..2d7ddb2 100644 --- a/core/src/main/scala/codesearch/core/syntax/stream.scala +++ b/core/src/main/scala/codesearch/core/syntax/stream.scala @@ -5,11 +5,13 @@ import fs2.Stream import cats.syntax.functor._ object stream { - implicit class StreamOps[F[_]: Functor, A](val stream: Stream[F, A]) { - def filterM(f: A => F[Boolean]): Stream[F, A] = + implicit class StreamOps[F[_], A](val stream: Stream[F, A]) { + def filterM(f: A => F[Boolean])(implicit F: Functor[F]): Stream[F, A] = stream.evalMap(x => f(x).map(x -> _)).collect { case (value, true) => value } - def filterNotM(f: A => F[Boolean]): Stream[F, A] = + def filterNotM(f: A => F[Boolean])(implicit F: Functor[F]): Stream[F, A] = stream.evalMap(x => f(x).map(x -> _)).collect { case (value, false) => value } + + def filterNot(f: A => Boolean): Stream[F, A] = stream.filter(!f(_)) } } diff --git a/web-server/app/codesearch/web/controllers/SearchController.scala b/web-server/app/codesearch/web/controllers/SearchController.scala index e39a6a9..602e1ba 100644 --- a/web-server/app/codesearch/web/controllers/SearchController.scala +++ b/web-server/app/codesearch/web/controllers/SearchController.scala @@ -42,13 +42,24 @@ trait SearchController[V <: DefaultTable] { self: InjectedController => spaceInsensitive: String, precise: String, sources: String, + withoutTests: String, page: String ): Action[AnyContent] = Action.async { implicit request => val host: String = request.host val searchRequest = - SearchRequest.applyRaw(lang, query, filter, filePath, caseInsensitive, spaceInsensitive, precise, sources, page) - + SearchRequest.applyRaw(lang, + query, + filter, + filePath, + caseInsensitive, + spaceInsensitive, + precise, + sources, + withoutTests, + page) + println(withoutTests) + println(searchRequest.withoutTests) db.updated.flatMap { updated => searchEngine.search(searchRequest) map { case CSearchPage(results, total) => @@ -63,6 +74,7 @@ trait SearchController[V <: DefaultTable] { self: InjectedController => space = searchRequest.spaceInsensitive, precise = searchRequest.preciseMatch, sources = searchRequest.sourcesOnly, + withoutTests = searchRequest.withoutTests, page = searchRequest.page, totalMatches = total, callURI = searchRequest.callURI(host).toString, diff --git a/web-server/app/views/docFrame.scala.html b/web-server/app/views/docFrame.scala.html index 1a07e21..3141185 100644 --- a/web-server/app/views/docFrame.scala.html +++ b/web-server/app/views/docFrame.scala.html @@ -45,6 +45,11 @@

Features

Uncheck this checkbox to search through everything contained in the packages, not just source files and package manifests. +
Without test
+
+ The search will not be conducted in tests. + * Sometimes incorrect results are possible and a search in tests will also be displayed. +

How it works

diff --git a/web-server/app/views/search.scala.html b/web-server/app/views/search.scala.html index 40ff536..0d115d1 100644 --- a/web-server/app/views/search.scala.html +++ b/web-server/app/views/search.scala.html @@ -5,6 +5,6 @@ } @wrapper(s"Codesearch | $lang", headExtra){ - @searchBox(s"/$lang/search", "", None, None, insensitive = false, space = false, precise = false, sources = true) + @searchBox(s"/$lang/search", "", None, None, insensitive = false, space = false, precise = false, sources = true, withoutTests = true) @docFrame() } diff --git a/web-server/app/views/searchBox.scala.html b/web-server/app/views/searchBox.scala.html index 6eb530e..e6b8cfa 100644 --- a/web-server/app/views/searchBox.scala.html +++ b/web-server/app/views/searchBox.scala.html @@ -6,7 +6,8 @@ insensitive: Boolean, space: Boolean, precise: Boolean, - sources: Boolean + sources: Boolean, + withoutTests: Boolean, )
@@ -40,6 +41,10 @@ +
+ + +
diff --git a/web-server/app/views/searchResults.scala.html b/web-server/app/views/searchResults.scala.html index c949666..09b385d 100644 --- a/web-server/app/views/searchResults.scala.html +++ b/web-server/app/views/searchResults.scala.html @@ -10,6 +10,7 @@ space: Boolean, precise: Boolean, sources: Boolean, + withoutTests: Boolean, page: Int, totalMatches: Int, callURI: String, @@ -23,7 +24,7 @@ } @wrapper(s"Codesearch | $lang", headExtra) { - @searchBox(s"/$lang/search", query, filter, filePath, insensitive, space, precise, sources) + @searchBox(s"/$lang/search", query, filter, filePath, insensitive, space, precise, sources, withoutTests) @resultFrame(lang, insensitive, space, precise, query, updated, packages, totalMatches) @pagination(page, totalMatches, callURI) } diff --git a/web-server/conf/routes b/web-server/conf/routes index 097b4ba..29a7b5e 100644 --- a/web-server/conf/routes +++ b/web-server/conf/routes @@ -3,19 +3,19 @@ GET / codesearch.web.controllers.Applicat GET /*path/ codesearch.web.controllers.Application.untrail(path: String) GET /haskell codesearch.web.controllers.HackageSearcher.index -GET /haskell/search codesearch.web.controllers.HackageSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", page: String ?= "1") +GET /haskell/search codesearch.web.controllers.HackageSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", withoutTests: String ?= "on", page: String ?= "1") GET /haskell/src/*relativePath codesearch.web.controllers.HackageSearcher.source(relativePath, query: String ?= "", L: Int ?= -1) GET /rust codesearch.web.controllers.CratesSearcher.index -GET /rust/search codesearch.web.controllers.CratesSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", page: String ?= "1") +GET /rust/search codesearch.web.controllers.CratesSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", withoutTests: String ?= "on", page: String ?= "1") GET /rust/src/*relativePath codesearch.web.controllers.CratesSearcher.source(relativePath, query: String ?= "", L: Int ?= -1) GET /js codesearch.web.controllers.NpmSearcher.index -GET /js/search codesearch.web.controllers.NpmSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", page: String ?= "1") +GET /js/search codesearch.web.controllers.NpmSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", withoutTests: String ?= "on", page: String ?= "1") GET /js/src/*relativePath codesearch.web.controllers.NpmSearcher.source(relativePath, query: String ?= "", L: Int ?= -1) GET /ruby codesearch.web.controllers.GemSearcher.index -GET /ruby/search codesearch.web.controllers.GemSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", page: String ?= "1") +GET /ruby/search codesearch.web.controllers.GemSearcher.search(query: String ?= "", filter: Option[String], filePath: Option[String], insensitive: String ?= "", space: String ?= "", precise: String ?= "", sources: String ?= "on", withoutTests: String ?= "on", page: String ?= "1") GET /ruby/src/*relativePath codesearch.web.controllers.GemSearcher.source(relativePath, query: String ?= "", L: Int ?= -1) GET /versionedAssets/*file controllers.Assets.versioned(path="/public", file: Asset) From 85d946124f3abfdc2f775dd5d966737227b0387f Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 26 Apr 2019 14:33:35 +0300 Subject: [PATCH 4/6] Added total matches in snippet info. --- core/src/main/scala/codesearch/core/search/Search.scala | 3 ++- .../scala/codesearch/core/search/SnippetsGrouper.scala | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index 3b93810..9c04a70 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -47,7 +47,8 @@ trait Search { .through(groupByPackage) .compile .toList - } yield CSearchPage(results.sortBy(_.pack.name), filteredSnippetsInfo.compile.toList.size) + totalMatches = filteredSnippetsInfo.fold(0)((total, snippet) => total + snippet.totalMatches).compile.toList.last + } yield CSearchPage(results.sortBy(_.pack.name), totalMatches) } /** diff --git a/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala b/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala index cc86523..680dba3 100644 --- a/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala +++ b/core/src/main/scala/codesearch/core/search/SnippetsGrouper.scala @@ -13,7 +13,7 @@ object SnippetsGrouper { * @param filePath absolute path to file * @param lines numbers of matched lines in file */ - case class SnippetInfo(filePath: String, lines: NonEmptyVector[Int]) + case class SnippetInfo(filePath: String, lines: NonEmptyVector[Int], totalMatches: Int) private case class ResultRow(path: String, lineNumber: Int) @@ -40,12 +40,13 @@ object SnippetsGrouper { snippets.lastOption match { case Some(snippet) => if (row.lineNumber < snippet.lines.last + config.linesAfter) { - snippets.init :+ snippet.copy(lines = snippet.lines :+ row.lineNumber) + snippets.init :+ snippet.copy(lines = snippet.lines :+ row.lineNumber, + totalMatches = snippet.totalMatches + 1) } else { - snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber)) + snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber), 1) } case None => - snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber)) + snippets :+ SnippetInfo(row.path, NonEmptyVector.one(row.lineNumber), 1) } } } From ba78e91e32768af696e72f52889fc49691b6b17d Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 26 Apr 2019 14:47:18 +0300 Subject: [PATCH 5/6] Edited tests --- .../scala/codesearch/core/search/Search.scala | 2 +- .../core/search/HaskellSearchSpec.scala | 8 ++++---- .../core/search/JavaScriptSearchSpec.scala | 8 ++++---- .../codesearch/core/search/RubySearchSpec.scala | 8 ++++---- .../codesearch/core/search/RustSearchSpec.scala | 8 ++++---- .../core/search/SnippetsGrouperSpec.scala | 15 ++++++++++----- 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/core/src/main/scala/codesearch/core/search/Search.scala b/core/src/main/scala/codesearch/core/search/Search.scala index 9c04a70..2c37f4a 100644 --- a/core/src/main/scala/codesearch/core/search/Search.scala +++ b/core/src/main/scala/codesearch/core/search/Search.scala @@ -12,7 +12,7 @@ import codesearch.core.index.repository.Extensions import codesearch.core.search.Search.{CSearchPage, CSearchResult, CodeSnippet, Package, PackageResult, snippetConfig} import codesearch.core.search.SnippetsGrouper.SnippetInfo import codesearch.core.util.Helper.readFileAsync -import fs2.{Pipe, Pure, Stream} +import fs2.{Pipe, Stream} import io.chrisdavenport.log4cats.SelfAwareStructuredLogger import io.chrisdavenport.log4cats.slf4j.Slf4jLogger import codesearch.core.regex.RegexConstructor diff --git a/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala b/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala index f1aae5d..99efef8 100644 --- a/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala +++ b/core/src/test/scala/codesearch/core/search/HaskellSearchSpec.scala @@ -2,9 +2,9 @@ package codesearch.core.search class HaskellSearchSpec extends SearchSpecBase { "Checking on the way to tests" - { - haskellIsTestInWay("path/to/package/test/", true) - haskellIsTestInWay("path/to/package/testSUites/", true) - haskellIsTestInWay("path/to/package/tEsts/", true) - haskellIsTestInWay("path/to/package/tes-t/", false) + haskellIsTestInWay(path = "path/to/package/test/", result = true) + haskellIsTestInWay(path = "path/to/package/testSUites/", result = true) + haskellIsTestInWay(path = "path/to/package/tEsts/", result = true) + haskellIsTestInWay(path = "path/to/package/tes-t/", result = false) } } diff --git a/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala b/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala index 529b792..f61345b 100644 --- a/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala +++ b/core/src/test/scala/codesearch/core/search/JavaScriptSearchSpec.scala @@ -2,9 +2,9 @@ package codesearch.core.search class JavaScriptSearchSpec extends SearchSpecBase { "Checking on the way to tests" - { - javaScriptIsTestInWay("path/to/package/tests/", true) - javaScriptIsTestInWay("path/to/package/sPEc-class/", true) - javaScriptIsTestInWay("path/to/package/tEsts.js", true) - javaScriptIsTestInWay("path/to/package/tes-t/", false) + javaScriptIsTestInWay(path = "path/to/package/tests/", result = true) + javaScriptIsTestInWay(path = "path/to/package/sPEc-class/", result = true) + javaScriptIsTestInWay(path = "path/to/package/tEsts.js", result = true) + javaScriptIsTestInWay(path = "path/to/package/tes-t/", result = false) } } diff --git a/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala b/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala index a6ced1c..f9a9e20 100644 --- a/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala +++ b/core/src/test/scala/codesearch/core/search/RubySearchSpec.scala @@ -2,9 +2,9 @@ package codesearch.core.search class RubySearchSpec extends SearchSpecBase { "Checking on the way to tests" - { - rubyIsTestInWay("path/to/package/test/", true) - rubyIsTestInWay("path/to/package/spec/", true) - rubyIsTestInWay("path/to/package/tEsts/", false) - rubyIsTestInWay("path/to/package/tes-t/", false) + rubyIsTestInWay(path = "path/to/package/test/", result = true) + rubyIsTestInWay(path = "path/to/package/spec/", result = true) + rubyIsTestInWay(path = "path/to/package/tEsts/", result = false) + rubyIsTestInWay(path = "path/to/package/tes-t/", result = false) } } diff --git a/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala b/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala index 0ab34fc..cbd0501 100644 --- a/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala +++ b/core/src/test/scala/codesearch/core/search/RustSearchSpec.scala @@ -2,9 +2,9 @@ package codesearch.core.search class RustSearchSpec extends SearchSpecBase { "Checking on the way to tests" - { - rustIsTestInWay("path/to/package/test/", true) - rustIsTestInWay("path/to/package/spec/", false) - rustIsTestInWay("path/to/package/tEsts/", false) - rustIsTestInWay("path/to/package/tes-t/", false) + rustIsTestInWay(path = "path/to/package/tests/", result = true) + rustIsTestInWay(path = "path/to/package/spec/", result = false) + rustIsTestInWay(path = "path/to/package/tEsts/", result = false) + rustIsTestInWay(path = "path/to/package/tes-t/", result = false) } } diff --git a/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala b/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala index 14152a8..dcbf868 100644 --- a/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala +++ b/core/src/test/scala/codesearch/core/search/SnippetsGrouperSpec.scala @@ -28,7 +28,8 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { snippets should contain theSameElementsAs List( SnippetInfo( filePath = "3models/0.3.0/Graphics/Model/DirectX.hs", - lines = NonEmptyVector.of(13, 14, 15, 16, 17) + lines = NonEmptyVector.of(13, 14, 15, 16, 17), + totalMatches = 5 ) ) } @@ -51,11 +52,13 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { snippets should contain theSameElementsAs List( SnippetInfo( filePath = "3models/0.3.0/Graphics/Model/DirectX.hs", - lines = NonEmptyVector.of(28) + lines = NonEmptyVector.of(28), + totalMatches = 1 ), SnippetInfo( filePath = "3models/0.3.0/Graphics/Model/DirectX.hs", - lines = NonEmptyVector.of(39) + lines = NonEmptyVector.of(39), + totalMatches = 1 ) ) } @@ -78,11 +81,13 @@ class SnippetsGrouperSpec extends WordSpec with Matchers { snippets should contain theSameElementsAs List( SnippetInfo( filePath = "3models/0.3.0/Graphics/Model/DirectX.hs", - lines = NonEmptyVector.of(14) + lines = NonEmptyVector.of(14), + totalMatches = 1 ), SnippetInfo( filePath = "3models/0.3.0/Graphics/Model/Obj.hs", - lines = NonEmptyVector.of(16) + lines = NonEmptyVector.of(16), + totalMatches = 1 ) ) } From 8d35ce922deb63080cda3b29704e5e06fe977512 Mon Sep 17 00:00:00 2001 From: yan Date: Tue, 13 Aug 2019 00:10:56 +0500 Subject: [PATCH 6/6] [#230] Excluding search in tests: - fixed tests --- .../codesearch/core/search/SearchSpecBase.scala | 15 +++++++++++---- .../integration/IntegrationHaskellSpec.scala | 8 +++++--- .../integration/IntegrationJavaScriptSpec.scala | 8 +++++--- .../scala/integration/IntegrationRubySpec.scala | 8 +++++--- .../scala/integration/IntegrationRustSpec.scala | 8 +++++--- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala b/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala index 66f6372..175a212 100644 --- a/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala +++ b/core/src/test/scala/codesearch/core/search/SearchSpecBase.scala @@ -1,13 +1,20 @@ package codesearch.core.search +import java.nio.file.Paths + +import codesearch.core.index.directory.{HaskellCindex, JavaScriptCindex, RubyCindex, RustCindex} import org.scalatest.{FreeSpec, Matchers} trait SearchSpecBase extends FreeSpec with Matchers{ - val haskellSearch = new HaskellSearch - val rubySearch = new RubySearch - val rustSearch = new RustSearch - val javaScriptSearch = new JavaScriptSearch + val haskellCindex = HaskellCindex(Paths.get("./index/test/cindex/")) + val haskellSearch = new HaskellSearch(haskellCindex) + val rubyCindex = RubyCindex(Paths.get("./index/test/cindex/")) + val rubySearch = new RubySearch(rubyCindex) + val rustCindex = RustCindex(Paths.get("./index/test/cindex/")) + val rustSearch = new RustSearch(rustCindex) + val javaScriptCindex = JavaScriptCindex(Paths.get("./index/test/cindex/")) + val javaScriptSearch = new JavaScriptSearch(javaScriptCindex) def haskellIsTestInWay(path: String, result: Boolean): Unit = { path in { diff --git a/core/src/test/scala/integration/IntegrationHaskellSpec.scala b/core/src/test/scala/integration/IntegrationHaskellSpec.scala index 48368db..2f1cc8d 100644 --- a/core/src/test/scala/integration/IntegrationHaskellSpec.scala +++ b/core/src/test/scala/integration/IntegrationHaskellSpec.scala @@ -11,9 +11,9 @@ import codesearch.core.meta._ import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} import codesearch.core.search.{HaskellSearch, Search, SearchRequest} import codesearch.core.util.Unarchiver +import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} import integration.fakes.FakeDownloader import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { @@ -48,7 +48,8 @@ class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with Inte spaceInsensitive = false, preciseMatch = false, sourcesOnly = false, - page = 1 + page = 1, + withoutTests = false ), 1, Seq( @@ -87,7 +88,8 @@ class IntegrationHaskellSpec extends FreeSpec with ForAllTestContainer with Inte spaceInsensitive = false, preciseMatch = false, sourcesOnly = true, - page = 1 + page = 1, + withoutTests = false ), 1, Seq( diff --git a/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala b/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala index 66a53fc..21bc302 100644 --- a/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala +++ b/core/src/test/scala/integration/IntegrationJavaScriptSpec.scala @@ -11,9 +11,9 @@ import codesearch.core.meta._ import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} import codesearch.core.search.{JavaScriptSearch, Search, SearchRequest} import codesearch.core.util.Unarchiver +import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} import integration.fakes.FakeDownloader import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { @@ -48,7 +48,8 @@ class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with I spaceInsensitive = true, preciseMatch = true, sourcesOnly = true, - page = 1 + page = 1, + withoutTests = false ), 2, Seq( @@ -89,7 +90,8 @@ class IntegrationJavaScriptSpec extends FreeSpec with ForAllTestContainer with I spaceInsensitive = false, preciseMatch = true, sourcesOnly = true, - page = 1 + page = 1, + withoutTests = false ), 1, Seq( diff --git a/core/src/test/scala/integration/IntegrationRubySpec.scala b/core/src/test/scala/integration/IntegrationRubySpec.scala index 2d0b48e..3affadc 100644 --- a/core/src/test/scala/integration/IntegrationRubySpec.scala +++ b/core/src/test/scala/integration/IntegrationRubySpec.scala @@ -11,9 +11,9 @@ import codesearch.core.meta._ import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} import codesearch.core.search.{RubySearch, Search, SearchRequest} import codesearch.core.util.Unarchiver +import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} import integration.fakes.FakeDownloader import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { @@ -48,7 +48,8 @@ class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with Integra spaceInsensitive = false, preciseMatch = false, sourcesOnly = false, - page = 1 + page = 1, + withoutTests = false ), 1, Seq( @@ -85,7 +86,8 @@ class IntegrationRubySpec extends FreeSpec with ForAllTestContainer with Integra spaceInsensitive = false, preciseMatch = false, sourcesOnly = false, - page = 1 + page = 1, + withoutTests = false ), 2, Seq( diff --git a/core/src/test/scala/integration/IntegrationRustSpec.scala b/core/src/test/scala/integration/IntegrationRustSpec.scala index ec96a9e..3d98e26 100644 --- a/core/src/test/scala/integration/IntegrationRustSpec.scala +++ b/core/src/test/scala/integration/IntegrationRustSpec.scala @@ -11,9 +11,9 @@ import codesearch.core.meta._ import codesearch.core.search.Search.{CodeSnippet, Package, PackageResult} import codesearch.core.search.{RustSearch, Search, SearchRequest} import codesearch.core.util.Unarchiver +import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} import integration.fakes.FakeDownloader import org.scalatest.FreeSpec -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} class IntegrationRustSpec extends FreeSpec with ForAllTestContainer with IntegrationSpecBase { @@ -48,7 +48,8 @@ class IntegrationRustSpec extends FreeSpec with ForAllTestContainer with Integra spaceInsensitive = false, preciseMatch = true, sourcesOnly = false, - page = 1 + page = 1, + withoutTests = false ), 2, Seq( @@ -104,7 +105,8 @@ class IntegrationRustSpec extends FreeSpec with ForAllTestContainer with Integra spaceInsensitive = false, preciseMatch = true, sourcesOnly = true, - page = 1 + page = 1, + withoutTests = false ), 1, Seq(