Skip to content

Commit 7ad82b8

Browse files
lolgablolgab
committed
Fix timing issues waiting for scala-cli linking
Before there were two concurrent proesses - one was publishing a browser refresh signal when linking was done - one was watching the file-system for changes and checking updating the map of hashes The problem is that sometimes the event for changed file comes after linking has finished, creating an issue Now linking publishes an event in `linkingTopic`. `fileWatcher` waits reacts to events in `linkingTopic` by listing all files in the directory, calculating the hash and then publishing an event in `refreshTopic` which refreshes the browser
1 parent 0557bb0 commit 7ad82b8

File tree

5 files changed

+71
-79
lines changed

5 files changed

+71
-79
lines changed

‎project/src/build.runner.scala‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,31 +16,33 @@ import cats.effect.IO
1616
import cats.effect.OutcomeIO
1717
import cats.effect.ResourceIO
1818

19+
import scala.concurrent.duration.*
20+
1921
sealed trait BuildTool
2022
class ScalaCli extends BuildTool
2123
class Mill extends BuildTool
2224

2325
def buildRunner(
2426
tool: BuildTool,
25-
refreshTopic: Topic[IO, String],
27+
linkingTopic: Topic[IO, Unit],
2628
workDir: fs2.io.file.Path,
2729
outDir: fs2.io.file.Path,
2830
extraBuildArgs: List[String],
2931
millModuleName: Option[String]
3032
)(
3133
logger: Scribe[IO]
3234
): ResourceIO[IO[OutcomeIO[Unit]]] = tool match
33-
case scli: ScalaCli => buildRunnerScli(refreshTopic, workDir, outDir, extraBuildArgs)(logger)
35+
case scli: ScalaCli => buildRunnerScli(linkingTopic, workDir, outDir, extraBuildArgs)(logger)
3436
case m: Mill =>
3537
buildRunnerMill(
36-
refreshTopic,
38+
linkingTopic,
3739
workDir,
3840
millModuleName.getOrElse(throw new Exception("must have a module name when running with mill")),
3941
extraBuildArgs
4042
)(logger)
4143

4244
def buildRunnerScli(
43-
refreshTopic: Topic[IO, String],
45+
linkingTopic: Topic[IO, Unit],
4446
workDir: fs2.io.file.Path,
4547
outDir: fs2.io.file.Path,
4648
extraBuildArgs: List[String]
@@ -79,7 +81,7 @@ def buildRunnerScli(
7981
aChunk =>
8082
if aChunk.toString.contains("main.js, run it with") then
8183
logger.trace("Detected that linking was successful, emitting refresh event") >>
82-
refreshTopic.publish1("refresh")
84+
linkingTopic.publish1(())
8385
else
8486
logger.trace(s"$aChunk :: Linking unfinished") >>
8587
IO.unit
@@ -92,7 +94,7 @@ def buildRunnerScli(
9294
end buildRunnerScli
9395

9496
def buildRunnerMill(
95-
refreshTopic: Topic[IO, String],
97+
linkingTopic: Topic[IO, Unit],
9698
workDir: fs2.io.file.Path,
9799
moduleName: String,
98100
extraBuildArgs: List[String]
@@ -105,15 +107,15 @@ def buildRunnerMill(
105107
.Stream
106108
.resource(Watcher.default[IO].evalTap(_.watch(watchLinkComplePath.toNioPath)))
107109
.flatMap {
108-
_.events()
110+
_.events(100.millis)
109111
.evalTap {
110112
(e: Event) =>
111113
e match
112114
case Created(path, count) => logger.info("fastLinkJs.json was created")
113115
case Deleted(path, count) => logger.info("fastLinkJs.json was deleted")
114116
case Modified(path, count) =>
115117
logger.info("fastLinkJs.json was modified - link successful => trigger a refresh") >>
116-
refreshTopic.publish1("refresh")
118+
linkingTopic.publish1(())
117119
case Overflow(count) => logger.info("overflow")
118120
case NonStandard(event, registeredDirectory) => logger.info("non-standard")
119121

‎project/src/hasher.scala‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,11 @@
1-
import java.security.MessageDigest
2-
31
import cats.effect.IO
2+
import fs2.io.file.*
43

5-
val md = MessageDigest.getInstance("MD5")
4+
// TODO: Use last modified time once scala-cli stops
5+
// copy pasting the files from a temporary directory
6+
// and performs linking in place
7+
// def fileHash(filePath: Path): IO[String] =
8+
// Files[IO].getLastModifiedTime(filePath).map(_.toNanos.toString)
69

7-
def fielHash(filePath: fs2.io.file.Path): IO[String] =
8-
fs2
9-
.io
10-
.file
11-
.Files[IO]
12-
.readUtf8Lines(filePath)
13-
.compile
14-
.toList
15-
.map(lines => md.digest(lines.mkString("\n").getBytes).map("%02x".format(_)).mkString)
10+
def fileHash(filePath: fs2.io.file.Path): IO[String] =
11+
Files[IO].readAll(filePath).through(fs2.hash.md5).through(fs2.text.hex.encode).compile.string

‎project/src/live.server.scala‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -267,14 +267,15 @@ object LiveServer
267267

268268
fileToHashRef <- Ref[IO].of(Map.empty[String, String]).toResource
269269
fileToHashMapRef = MapRef.fromSingleImmutableMapRef[IO, String, String](fileToHashRef)
270-
refreshPub <- Topic[IO, String].toResource
270+
refreshTopic <- Topic[IO, Unit].toResource
271+
linkingTopic <- Topic[IO, Unit].toResource
271272
client <- EmberClientBuilder.default[IO].build
272273

273274
proxyRoutes: HttpRoutes[IO] <- makeProxyRoutes(client, pathPrefix, proxyConfig)
274275

275276
_ <- buildRunner(
276277
buildTool,
277-
refreshPub,
278+
linkingTopic,
278279
fs2.io.file.Path(baseDir),
279280
fs2.io.file.Path(outDir),
280281
extraBuildArgs,
@@ -283,11 +284,13 @@ object LiveServer
283284

284285
indexHtmlTemplate = externalIndexHtmlTemplate.getOrElse(vanillaTemplate(stylesDir.isDefined).render)
285286

286-
app <- routes(outDir.toString(), refreshPub, stylesDir, proxyRoutes, indexHtmlTemplate, fileToHashRef)(logger)
287+
app <- routes(outDir.toString(), refreshTopic, stylesDir, proxyRoutes, indexHtmlTemplate, fileToHashRef)(
288+
logger
289+
)
287290

288291
_ <- seedMapOnStart(outDir, fileToHashMapRef)(logger)
289292
// _ <- stylesDir.fold(Resource.unit)(sd => seedMapOnStart(sd, mr))
290-
_ <- fileWatcher(fs2.io.file.Path(outDir), fileToHashMapRef)(logger)
293+
_ <- fileWatcher(fs2.io.file.Path(outDir), fileToHashMapRef, linkingTopic, refreshTopic)(logger)
291294
// _ <- stylesDir.fold(Resource.unit[IO])(sd => fileWatcher(fs2.io.file.Path(sd), mr))
292295
_ <- logger.info(s"Start dev server on http://localhost:$port").toResource
293296
server <- buildServer(app, port)

‎project/src/routes.scala‎

Lines changed: 26 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ import org.typelevel.ci.CIStringSyntax
1616

1717
import fs2.*
1818
import fs2.concurrent.Topic
19-
import fs2.io.Watcher
20-
import fs2.io.Watcher.Event
2119
import fs2.io.file.Files
2220

2321
import scribe.Scribe
@@ -29,7 +27,6 @@ import cats.effect.IO
2927
import cats.effect.kernel.Ref
3028
import cats.effect.kernel.Resource
3129
import cats.effect.std.*
32-
import cats.effect.std.MapRef
3330
import cats.syntax.all.*
3431

3532
import _root_.io.circe.syntax.EncoderOps
@@ -89,7 +86,7 @@ end ETagMiddleware
8986

9087
def routes(
9188
stringPath: String,
92-
refreshTopic: Topic[IO, String],
89+
refreshTopic: Topic[IO, Unit],
9390
stylesPath: Option[String],
9491
proxyRoutes: HttpRoutes[IO],
9592
indexHtmlTemplate: String,
@@ -162,7 +159,7 @@ def seedMapOnStart(stringPath: String, mr: MapRef[IO, String, Option[String]])(l
162159
.isRegularFile(f)
163160
.ifM(
164161
// logger.trace(s"hashing $f") >>
165-
fielHash(f).flatMap(
162+
fileHash(f).flatMap(
166163
h =>
167164
val key = asFs2.relativize(f)
168165
logger.trace(s"hashing $f to put at $key with hash : $h") >>
@@ -179,46 +176,33 @@ end seedMapOnStart
179176

180177
private def fileWatcher(
181178
stringPath: fs2.io.file.Path,
182-
mr: MapRef[IO, String, Option[String]]
183-
)(logger: Scribe[IO]): ResourceIO[IO[OutcomeIO[Unit]]] =
184-
fs2
185-
.Stream
186-
.resource(Watcher.default[IO].evalTap(_.watch(stringPath.toNioPath)))
187-
.flatMap {
188-
w =>
189-
w.events()
190-
.evalTap(
191-
(e: Event) =>
192-
e match
193-
case Event.Created(path, i) =>
194-
// if path.endsWith(".js") then
195-
logger.trace(s"created $path, calculating hash") >>
196-
fielHash(fs2.io.file.Path(path.toString())).flatMap(
197-
h =>
198-
val serveAt = stringPath.relativize(fs2.io.file.Path(path.toString()))
199-
logger.trace(s"$serveAt :: hash -> $h") >>
200-
mr.setKeyValue(serveAt.toString(), h)
201-
)
202-
// else IO.unit
203-
case Event.Modified(path, i) =>
204-
// if path.endsWith(".js") then
205-
logger.trace(s"modified $path, calculating hash") >>
206-
fielHash(fs2.io.file.Path(path.toString())).flatMap(
207-
h =>
208-
val serveAt = stringPath.relativize(fs2.io.file.Path(path.toString()))
209-
logger.trace(s"$serveAt :: hash -> $h") >>
210-
mr.setKeyValue(serveAt.toString(), h)
211-
)
212-
// else IO.unit
213-
case Event.Deleted(path, i) =>
179+
mr: MapRef[IO, String, Option[String]],
180+
linkingTopic: Topic[IO, Unit],
181+
refreshTopic: Topic[IO, Unit]
182+
)(logger: Scribe[IO]): ResourceIO[Unit] =
183+
linkingTopic
184+
.subscribe(10)
185+
.evalTap {
186+
_ =>
187+
fs2
188+
.io
189+
.file
190+
.Files[IO]
191+
.list(stringPath)
192+
.evalTap {
193+
path =>
194+
fileHash(path).flatMap(
195+
h =>
214196
val serveAt = stringPath.relativize(fs2.io.file.Path(path.toString()))
215-
logger.trace(s"deleted $path, removing key $serveAt") >>
216-
mr.unsetKey(serveAt.toString())
217-
case e: Event.Overflow => logger.info("overflow")
218-
case e: Event.NonStandard => logger.info("non-standard")
219-
)
197+
logger.trace(s"$serveAt :: hash -> $h") >>
198+
mr.setKeyValue(serveAt.toString(), h)
199+
)
200+
}
201+
.compile
202+
.drain >> refreshTopic.publish1(())
220203
}
221204
.compile
222205
.drain
223206
.background
207+
.void
224208
end fileWatcher

‎project/test/src/RoutesSpec.scala‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import java.security.MessageDigest
22

3-
import scala.concurrent.duration.DurationInt
4-
53
import org.http4s.HttpRoutes
64
import org.typelevel.ci.CIStringSyntax
75

@@ -13,7 +11,8 @@ import cats.effect.kernel.Ref
1311
import cats.effect.std.MapRef
1412

1513
import munit.CatsEffectSuite
16-
// import cats.effect.unsafe.implicits.global
14+
15+
import scala.concurrent.duration.*
1716

1817
class ExampleSuite extends CatsEffectSuite:
1918

@@ -56,22 +55,30 @@ class ExampleSuite extends CatsEffectSuite:
5655

5756
files.test("watched map is updated") {
5857
tempDir =>
59-
val newStr = "const hi = 'bye, world'"
60-
val newHash = md.digest(testStr.getBytes()).map("%02x".format(_)).mkString
6158
val toCheck = for
6259
logger <- IO(scribe.cats[IO]).toResource
6360
fileToHashRef <- Ref[IO].of(Map.empty[String, String]).toResource
61+
linkingTopic <- Topic[IO, Unit].toResource
62+
refreshTopic <- Topic[IO, Unit].toResource
6463
fileToHashMapRef = MapRef.fromSingleImmutableMapRef[IO, String, String](fileToHashRef)
64+
_ <- fileWatcher(fs2.io.file.Path(tempDir.toString), fileToHashMapRef, linkingTopic, refreshTopic)(logger)
65+
_ <- IO.sleep(100.millis).toResource // wait for watcher to start
6566
_ <- seedMapOnStart(tempDir.toString, fileToHashMapRef)(logger)
66-
_ <- fileWatcher(fs2.io.file.Path(tempDir.toString), fileToHashMapRef)(logger)
67-
_ <- IO(os.write.over(tempDir / "test.js", newStr)).toResource
68-
_ <- IO.sleep(1.second).toResource
69-
updatedMap <- fileToHashRef.get.toResource
70-
yield updatedMap
67+
_ <- IO.blocking(os.write.over(tempDir / "test.js", "const hi = 'bye, world'")).toResource
68+
_ <- linkingTopic.publish1(()).toResource
69+
_ <- refreshTopic.subscribe(1).head.compile.resource.drain
70+
oldHash <- fileToHashRef.get.map(_("test.js")).toResource
71+
_ <- IO.blocking(os.write.over(tempDir / "test.js", "const hi = 'modified, workd'")).toResource
72+
_ <- linkingTopic.publish1(()).toResource
73+
_ <- refreshTopic.subscribe(1).head.compile.resource.drain
74+
newHash <- fileToHashRef.get.map(_("test.js")).toResource
75+
yield oldHash -> newHash
7176

7277
toCheck.use {
73-
updatedMap =>
74-
assertIO(IO(updatedMap.get("test.js")), Some(newHash))
78+
case (oldHash, newHash) =>
79+
IO(assertNotEquals(oldHash, newHash)) >>
80+
IO(assertEquals(oldHash, "27b2d040a66fb938f134c4b66fb7e9ce")) >>
81+
IO(assertEquals(newHash, "3ebb82d4d6236c6bfbb90d65943b3e3d"))
7582
}
7683

7784
}
@@ -84,7 +91,7 @@ class ExampleSuite extends CatsEffectSuite:
8491
fileToHashRef <- Ref[IO].of(Map.empty[String, String]).toResource
8592
fileToHashMapRef = MapRef.fromSingleImmutableMapRef[IO, String, String](fileToHashRef)
8693
_ <- seedMapOnStart(tempDir.toString, fileToHashMapRef)(logger)
87-
refreshPub <- Topic[IO, String].toResource
94+
refreshPub <- Topic[IO, Unit].toResource
8895
theseRoutes <- routes(
8996
tempDir.toString,
9097
refreshPub,

0 commit comments

Comments
 (0)