Skip to content

Commit 31ab44d

Browse files
m-dzianishchytsm-dzianishchyts
committed
Fix CI hanging
1 parent 38e2c76 commit 31ab44d

15 files changed

Lines changed: 121 additions & 82 deletions

File tree

‎jjava-distro/src/test/java/org/dflib/jjava/distro/ContainerizedKernelCase.java‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,20 @@
1313
import java.io.IOException;
1414
import java.time.Duration;
1515
import java.util.ArrayList;
16-
import java.util.Base64;
16+
import java.util.Arrays;
1717
import java.util.Collections;
1818
import java.util.List;
1919
import java.util.Map;
20+
import java.util.stream.Collectors;
21+
import java.util.stream.Stream;
2022

2123
import static org.junit.jupiter.api.Assertions.assertEquals;
2224

2325
public abstract class ContainerizedKernelCase {
2426

2527
private static final Logger LOGGER = LoggerFactory.getLogger(ContainerizedKernelCase.class);
2628

27-
protected static final GenericContainer<?> container;
29+
protected static GenericContainer<?> container;
2830
protected static final String WORKING_DIRECTORY = "/test";
2931
protected static final String CONTAINER_KERNELSPEC = "/usr/share/jupyter/kernels/java";
3032
protected static final String CONTAINER_RESOURCES = WORKING_DIRECTORY + "/resources";
@@ -34,20 +36,10 @@ public abstract class ContainerizedKernelCase {
3436
private static final String FS_KERNELSPEC = "../kernelspec/java";
3537
private static final String FS_RESOURCES = "src/test/resources";
3638

37-
static {
38-
container = new GenericContainer<>(BASE_IMAGE)
39-
.withWorkingDirectory(WORKING_DIRECTORY)
40-
.withCopyToContainer(MountableFile.forHostPath(FS_KERNELSPEC), CONTAINER_KERNELSPEC)
41-
.withCopyToContainer(MountableFile.forHostPath(FS_RESOURCES), CONTAINER_RESOURCES)
42-
.withCommand("bash", "-c", getStartupCommand())
43-
.withLogConsumer(new Slf4jLogConsumer(LOGGER))
44-
.waitingFor(Wait.forSuccessfulCommand(getSuccessfulCommand()))
45-
.withStartupTimeout(Duration.ofMinutes(1));
46-
container.start();
47-
}
48-
4939
@BeforeAll
50-
static void compileSources() throws IOException, InterruptedException {
40+
static void setUp() throws IOException, InterruptedException {
41+
initializeContainer();
42+
5143
String source = "$(find " + CONTAINER_RESOURCES + "/src -name '*.java')";
5244
Container.ExecResult compileResult = executeInContainer("javac -d " + TEST_CLASSPATH + " " + source);
5345

@@ -68,9 +60,29 @@ protected static Container.ExecResult executeInKernel(String snippet) throws IOE
6860
}
6961

7062
protected static Container.ExecResult executeInKernel(String snippet, Map<String, String> env) throws IOException, InterruptedException {
71-
String snippet64 = Base64.getEncoder().encodeToString(snippet.getBytes());
72-
String jupyterCommand = venvCommand("jupyter console --kernel=java --simple-prompt");
73-
String[] containerCommand = new String[]{"bash", "-c", "echo \"" + snippet64 + "\" | base64 -d | " + jupyterCommand};
63+
long snippetLines = snippet.lines().count();
64+
String snippetEscaped = snippet.replace("\\", "\\\\").replace("\"", "\\\"");
65+
String snippetFeeding = Arrays.stream(snippetEscaped.split("\n"))
66+
.flatMap(line -> Stream.of(
67+
"p.expect(r'In \\[\\d+\\]:')",
68+
"p.sendline(\"" + line + "\")"
69+
))
70+
.collect(Collectors.joining("\n"));
71+
72+
String pexpectScript = String.join("\n",
73+
"import pexpect, sys, os, time",
74+
"env = os.environ.copy()",
75+
"env['PROMPT_TOOLKIT_NO_CPR'] = '1'",
76+
"env['TERM'] = 'dumb'",
77+
"p=pexpect.spawn('" + venvCommand("jupyter") + "', "
78+
+ "['console', '--kernel=java', '--no-confirm-exit'], "
79+
+ "env=env, timeout=60, encoding='utf-8')",
80+
"p.logfile_read = sys.stdout",
81+
snippetFeeding,
82+
"p.expect(r'In \\[" + (snippetLines + 1) + "\\]:')",
83+
"p.close(force=True)"
84+
);
85+
String[] containerCommand = new String[]{venvCommand("python"), "-c", pexpectScript};
7486
Container.ExecResult execResult = container.execInContainer(ExecConfig.builder()
7587
.envVars(env)
7688
.command(containerCommand)
@@ -80,17 +92,29 @@ protected static Container.ExecResult executeInKernel(String snippet, Map<String
8092
LOGGER.info("env = {}", env);
8193
LOGGER.info("snippet = {}", snippet);
8294
LOGGER.info("exitCode = {}", execResult.getExitCode());
83-
LOGGER.debug("stderr = {}", execResult.getStderr());
8495
LOGGER.debug("stdout = {}", execResult.getStdout());
96+
LOGGER.debug("stderr = {}", execResult.getStderr());
8597
return execResult;
8698
}
8799

100+
private static void initializeContainer() {
101+
container = new GenericContainer<>(BASE_IMAGE)
102+
.withWorkingDirectory(WORKING_DIRECTORY)
103+
.withCopyToContainer(MountableFile.forHostPath(FS_KERNELSPEC), CONTAINER_KERNELSPEC)
104+
.withCopyToContainer(MountableFile.forHostPath(FS_RESOURCES), CONTAINER_RESOURCES)
105+
.withCommand("bash", "-c", getStartupCommand())
106+
.withLogConsumer(new Slf4jLogConsumer(LOGGER))
107+
.waitingFor(Wait.forSuccessfulCommand(getSuccessfulCommand()))
108+
.withStartupTimeout(Duration.ofMinutes(1));
109+
container.start();
110+
}
111+
88112
private static String getStartupCommand() {
89113
return String.join(" && ",
90114
"apt-get update",
91-
"apt-get install --no-install-recommends -y python3 python3-pip python3-venv",
115+
"apt-get install --no-install-recommends -y python3 python3-pip python3-venv curl",
92116
"python3 -m venv ./venv",
93-
venvCommand("pip install jupyter-console --progress-bar off"),
117+
venvCommand("pip install jupyter-console pexpect --progress-bar off"),
94118
"tail -f /dev/null"
95119
);
96120
}

‎jjava-distro/src/test/java/org/dflib/jjava/distro/KernelEnvIT.java‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
11
package org.dflib.jjava.distro;
22

3-
import org.hamcrest.CoreMatchers;
43
import org.junit.jupiter.api.Test;
54
import org.testcontainers.containers.Container;
65

76
import java.util.Map;
87

9-
import static org.hamcrest.CoreMatchers.containsString;
10-
import static org.hamcrest.CoreMatchers.not;
118
import static org.hamcrest.MatcherAssert.assertThat;
9+
import static org.hamcrest.Matchers.allOf;
10+
import static org.hamcrest.Matchers.containsString;
11+
import static org.hamcrest.Matchers.not;
12+
import static org.junit.jupiter.api.Assertions.assertEquals;
1213

13-
public class KernelEnvIT extends ContainerizedKernelCase {
14+
class KernelEnvIT extends ContainerizedKernelCase {
1415

1516
@Test
16-
public void compilerOpts() throws Exception {
17+
void compilerOpts() throws Exception {
1718
Map<String, String> env = Map.of(Env.JJAVA_COMPILER_OPTS, "-source 9");
1819
String snippet = "var value = 1;";
1920
Container.ExecResult snippetResult = executeInKernel(snippet, env);
2021

21-
assertThat(snippetResult.getStderr(), CoreMatchers.allOf(
22+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
23+
assertThat(snippetResult.getStdout(), allOf(
2224
containsString("| var value = 1;"),
2325
containsString(Runtime.version().feature() == 11
2426
? "'var' is a restricted local variable type"
@@ -27,78 +29,85 @@ public void compilerOpts() throws Exception {
2729
}
2830

2931
@Test
30-
public void timeout() throws Exception {
32+
void timeout() throws Exception {
3133
Map<String, String> env = Map.of(Env.JJAVA_TIMEOUT, "3000");
3234
String snippet = "Thread.sleep(5000);";
3335
Container.ExecResult snippetResult = executeInKernel(snippet, env);
3436

35-
assertThat(snippetResult.getStderr(), CoreMatchers.allOf(
37+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
38+
assertThat(snippetResult.getStdout(), allOf(
3639
containsString("| " + snippet),
3740
containsString("Evaluation timed out after 3000 milliseconds.")
3841
));
3942
}
4043

4144
@Test
42-
public void classpath() throws Exception {
45+
void classpath() throws Exception {
4346
Map<String, String> env = Map.of(Env.JJAVA_CLASSPATH, TEST_CLASSPATH);
4447
String snippet = String.join("\n",
4548
"import org.dflib.jjava.Dummy;",
46-
"Dummy.class.getName()"
49+
"\"className = \" + Dummy.class.getName();"
4750
);
4851
Container.ExecResult snippetResult = executeInKernel(snippet, env);
4952

50-
assertThat(snippetResult.getStderr(), not(containsString("|")));
51-
assertThat(snippetResult.getStdout(), containsString("org.dflib.jjava.Dummy"));
53+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
54+
assertThat(snippetResult.getStdout(), not(containsString("|")));
55+
assertThat(snippetResult.getStdout(), containsString("className = org.dflib.jjava.Dummy"));
5256
}
5357

5458
@Test
55-
public void startUpScriptsPath() throws Exception {
56-
Map<String, String> env = Map.of(Env.JJAVA_STARTUP_SCRIPTS_PATH, CONTAINER_RESOURCES + "/test-ping.jshell");
59+
void startUpScriptsPath() throws Exception {
60+
Map<String, String> env = Map.of(Env.JJAVA_STARTUP_SCRIPTS_PATH, CONTAINER_RESOURCES + "/test-ping.jshell");
5761
String snippet = "ping()";
5862
Container.ExecResult snippetResult = executeInKernel(snippet, env);
5963

60-
assertThat(snippetResult.getStderr(), not(containsString("|")));
64+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
65+
assertThat(snippetResult.getStdout(), not(containsString("|")));
6166
assertThat(snippetResult.getStdout(), containsString("pong!"));
6267
}
6368

6469
@Test
65-
public void startUpScript() throws Exception {
70+
void startUpScript() throws Exception {
6671
Map<String, String> env = Map.of(Env.JJAVA_STARTUP_SCRIPT, "public String ping() { return \"pong!\"; }");
6772
String snippet = "ping()";
6873
Container.ExecResult snippetResult = executeInKernel(snippet, env);
6974

70-
assertThat(snippetResult.getStderr(), not(containsString("|")));
75+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
76+
assertThat(snippetResult.getStdout(), not(containsString("|")));
7177
assertThat(snippetResult.getStdout(), containsString("pong!"));
7278
}
7379

7480
@Test
75-
public void loadExtensions_Default() throws Exception {
81+
void loadExtensions_Default() throws Exception {
7682
String snippet = "printf(\"Hello, %s!\", \"world\");";
7783
Container.ExecResult snippetResult = executeInKernel(snippet);
7884

79-
assertThat(snippetResult.getStderr(), not(containsString("|")));
85+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
86+
assertThat(snippetResult.getStdout(), not(containsString("|")));
8087
assertThat(snippetResult.getStdout(), containsString("Hello, world!"));
8188
}
8289

8390
@Test
84-
public void loadExtensions_Disable() throws Exception {
91+
void loadExtensions_Disable() throws Exception {
8592
Map<String, String> env = Map.of(Env.JJAVA_LOAD_EXTENSIONS, "0");
8693
String snippet = "printf(\"Hello, %s!\", \"world\");";
8794
Container.ExecResult snippetResult = executeInKernel(snippet, env);
8895

89-
assertThat(snippetResult.getStderr(), CoreMatchers.allOf(
96+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
97+
assertThat(snippetResult.getStdout(), allOf(
9098
containsString("| " + snippet),
9199
containsString("cannot find symbol")
92100
));
93101
}
94102

95103
@Test
96-
public void jvmOpts() throws Exception {
104+
void jvmOpts() throws Exception {
97105
Map<String, String> env = Map.of(Env.JJAVA_JVM_OPTS, "-Xmx300m");
98106
String snippet = "Runtime.getRuntime().maxMemory()";
99107
Container.ExecResult snippetResult = executeInKernel(snippet, env);
100108

101-
assertThat(snippetResult.getStderr(), not(containsString("|")));
109+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
110+
assertThat(snippetResult.getStdout(), not(containsString("|")));
102111
assertThat(snippetResult.getStdout(), containsString(String.valueOf(300 * (int) Math.pow(1024, 2))));
103112
}
104113
}

‎jjava-distro/src/test/java/org/dflib/jjava/distro/KernelMagicIT.java‎

Lines changed: 24 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,23 @@
33
import org.junit.jupiter.api.Test;
44
import org.testcontainers.containers.Container;
55

6-
import static org.hamcrest.CoreMatchers.containsString;
7-
import static org.hamcrest.CoreMatchers.not;
86
import static org.hamcrest.MatcherAssert.assertThat;
7+
import static org.hamcrest.Matchers.containsString;
8+
import static org.hamcrest.Matchers.not;
99
import static org.junit.jupiter.api.Assertions.assertEquals;
1010

11-
public class KernelMagicIT extends ContainerizedKernelCase {
11+
class KernelMagicIT extends ContainerizedKernelCase {
1212

1313
@Deprecated
1414
@Test
15-
public void jars() throws Exception {
15+
void jars() throws Exception {
1616
String jar = CONTAINER_RESOURCES + "/jakarta.annotation-api-3.0.0.jar";
1717
Container.ExecResult fetchResult = container.execInContainer(
1818
"curl", "-L", "-s", "-S", "-f",
1919
"https://repo1.maven.org/maven2/jakarta/annotation/jakarta.annotation-api/3.0.0/jakarta.annotation-api-3.0.0.jar",
2020
"-o", jar
2121
);
22-
assertEquals("", fetchResult.getStderr());
22+
assertEquals(0, fetchResult.getExitCode(), fetchResult.getStdout());
2323

2424
String snippet = String.join("\n",
2525
"%jars " + jar,
@@ -28,64 +28,69 @@ public void jars() throws Exception {
2828
);
2929
Container.ExecResult snippetResult = executeInKernel(snippet);
3030

31-
assertThat(snippetResult.getStderr(), not(containsString("|")));
31+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
32+
assertThat(snippetResult.getStdout(), not(containsString("|")));
3233
assertThat(snippetResult.getStdout(), containsString("jakarta.annotation.Nullable"));
3334
}
3435

3536
@Test
36-
public void classpath() throws Exception {
37+
void classpath() throws Exception {
3738
String snippet = String.join("\n",
3839
"%classpath " + TEST_CLASSPATH,
3940
"import org.dflib.jjava.Dummy;",
40-
"Dummy.class.getName()"
41+
"\"className = \" + Dummy.class.getName();"
4142
);
4243
Container.ExecResult snippetResult = executeInKernel(snippet);
4344

44-
assertThat(snippetResult.getStderr(), not(containsString("|")));
45-
assertThat(snippetResult.getStdout(), containsString("org.dflib.jjava.Dummy"));
45+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
46+
assertThat(snippetResult.getStdout(), not(containsString("|")));
47+
assertThat(snippetResult.getStdout(), containsString("className = org.dflib.jjava.Dummy"));
4648
}
4749

4850
@Test
49-
public void maven() throws Exception {
51+
void maven() throws Exception {
5052
String snippet = String.join("\n",
5153
"%maven org.dflib:dflib-jupyter:1.0.0-RC1",
5254
"System.getProperty(\"java.class.path\")"
5355
);
5456
Container.ExecResult snippetResult = executeInKernel(snippet);
5557

56-
assertThat(snippetResult.getStderr(), not(containsString("|")));
58+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
59+
assertThat(snippetResult.getStdout(), not(containsString("|")));
5760
assertThat(snippetResult.getStdout(), containsString("dflib-jupyter-1.0.0-RC1.jar"));
5861
}
5962

6063
@Deprecated
6164
@Test
62-
public void mavenIvySyntax() throws Exception {
65+
void mavenIvySyntax() throws Exception {
6366
String snippet = String.join("\n",
6467
"%maven jakarta.annotation#jakarta.annotation-api;3.0.0",
6568
"System.getProperty(\"java.class.path\")"
6669
);
6770

6871
Container.ExecResult snippetResult = executeInKernel(snippet);
6972

70-
assertThat(snippetResult.getStderr(), not(containsString("|")));
73+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
74+
assertThat(snippetResult.getStdout(), not(containsString("|")));
7175
assertThat(snippetResult.getStdout(), containsString("jakarta.annotation-api-3.0.0.jar"));
7276
}
7377

7478
@Test
75-
public void load() throws Exception {
79+
void load() throws Exception {
7680
String script = CONTAINER_RESOURCES + "/test-ping.jshell";
7781
String snippet = String.join("\n",
7882
"%load " + script,
7983
"ping()"
8084
);
8185
Container.ExecResult snippetResult = executeInKernel(snippet);
8286

83-
assertThat(snippetResult.getStderr(), not(containsString("|")));
87+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
88+
assertThat(snippetResult.getStdout(), not(containsString("|")));
8489
assertThat(snippetResult.getStdout(), containsString("pong!"));
8590
}
8691

8792
@Test
88-
public void loadFromPOM() throws Exception {
93+
void loadFromPOM() throws Exception {
8994
String pom = CONTAINER_RESOURCES + "/test-pom.xml";
9095
String snippet = String.join("\n",
9196
"%loadFromPOM " + pom,
@@ -94,7 +99,8 @@ public void loadFromPOM() throws Exception {
9499
);
95100
Container.ExecResult snippetResult = executeInKernel(snippet);
96101

97-
assertThat(snippetResult.getStderr(), not(containsString("|")));
102+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
103+
assertThat(snippetResult.getStdout(), not(containsString("|")));
98104
assertThat(snippetResult.getStdout(), containsString("jakarta.annotation.Nullable"));
99105
}
100106
}

‎jjava-distro/src/test/java/org/dflib/jjava/distro/KernelStartupIT.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,21 @@
55

66
import java.util.Map;
77

8-
import static org.hamcrest.CoreMatchers.containsString;
9-
import static org.hamcrest.CoreMatchers.not;
108
import static org.hamcrest.MatcherAssert.assertThat;
9+
import static org.hamcrest.Matchers.containsString;
1110
import static org.hamcrest.Matchers.matchesPattern;
11+
import static org.hamcrest.Matchers.not;
12+
import static org.junit.jupiter.api.Assertions.assertEquals;
1213

13-
public class KernelStartupIT extends ContainerizedKernelCase {
14+
class KernelStartupIT extends ContainerizedKernelCase {
1415

1516
@Test
16-
public void startUp() throws Exception {
17+
void startUp() throws Exception {
1718
String snippet = "1000d + 1";
1819
Container.ExecResult snippetResult = executeInKernel(snippet);
1920

20-
assertThat(snippetResult.getStderr(), not(containsString("|")));
21+
assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout());
22+
assertThat(snippetResult.getStdout(), not(containsString("|")));
2123
assertThat(snippetResult.getStdout(), containsString("1001.0"));
2224
}
2325

0 commit comments

Comments
 (0)