GV

GeezerVison

The World Wide Web, restored to 800 pixels of glory.

Home|About|Gaming cabinet|Archives|How it works|Starter sites

On this site

  • Sign in
  • ActionsCI/CD and automation
  • CodespacesCloud development environments
  • All Features
  • Documentation
  • Training
  • Certifications
  • Build with Copilot
  • Customize Copilot
  • Integrations
  • Trending repos
  • GitHub Sponsors
  • Developer articles
  • Enterprises

This view

  • Original page
  • New address

GeezerVison is showing github.com — Use daemon for starting jupyter notebooks (#8543) · joyni/vscode-python@77866a1

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
  • news/1 Enhancements
    • 7242.md
  • pythonFiles/datascience
    • jupyter_daemon.py
  • src
    • client
      • common/process
        • pythonExecutionFactory.ts
      • datascience
        • activation.ts
        • interactive-ipynb
          • nativeEditorProvider.ts
        • jupyter
          • jupyterCommand.ts
          • jupyterCommandFinder.ts
          • jupyterExecution.ts
          • jupyterSession.ts
          • liveshare
            • hostJupyterServer.ts
          • notebookStarter.ts
        • types.ts
    • test
      • common/process
        • pythonExecutionFactory.unit.test.ts
      • datascience
        • activation.unit.test.ts
        • execution.unit.test.ts
        • mockJupyterManager.ts
        • testNativeEditorProvider.ts
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎news/1 Enhancements/7242.md‎

Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Improve startup performance of Jupyter by using a Python daemon.
Collapse file

‎pythonFiles/datascience/jupyter_daemon.py‎

Copy file name to clipboardExpand all lines: pythonFiles/datascience/jupyter_daemon.py
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ def m_exec_module(self, module_name, args=[], cwd=None, env=None):
3232
return self._execute_and_capture_output(self._print_kernel_list)
3333
elif module_name == "jupyter" and args == ["kernelspec", "--version"]:
3434
return self._execute_and_capture_output(self._print_kernelspec_version)
35+
elif module_name == "jupyter" and args[0] == "nbconvert" and args[-1] != "--version":
36+
return self._execute_and_capture_output(lambda : self._convert(args))
3537
else:
3638
log.info("check base class stuff")
3739
return super().m_exec_module(module_name, args, cwd, env)
@@ -87,6 +89,13 @@ def _print_kernel_list(self):
8789
)
8890
sys.stdout.flush()
8991

92+
def _convert(self, args):
93+
log.info("nbconvert")
94+
from nbconvert import nbconvertapp as app
95+
96+
sys.argv = [""] + args
97+
app.main()
98+
9099
def _start_notebook(self, args):
91100
from notebook import notebookapp as app
92101

Collapse file

‎src/client/common/process/pythonExecutionFactory.ts‎

Copy file name to clipboardExpand all lines: src/client/common/process/pythonExecutionFactory.ts
+6-3Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import { inject, injectable } from 'inversify';
44

55
import { IEnvironmentActivationService } from '../../interpreter/activation/types';
6+
import { IInterpreterService } from '../../interpreter/contracts';
67
import { WindowsStoreInterpreter } from '../../interpreter/locators/services/windowsStoreInterpreter';
78
import { IWindowsStoreInterpreter } from '../../interpreter/locators/types';
89
import { IServiceContainer } from '../../ioc/types';
@@ -52,14 +53,16 @@ export class PythonExecutionFactory implements IPythonExecutionFactory {
5253
const pythonPath = options.pythonPath ? options.pythonPath : this.configService.getSettings(options.resource).pythonPath;
5354
const daemonPoolKey = `${pythonPath}#${options.daemonClass || ''}#${options.daemonModule || ''}`;
5455
const disposables = this.serviceContainer.get<IDisposableRegistry>(IDisposableRegistry);
55-
const activatedProcPromise = this.createActivatedEnvironment({ allowEnvironmentFetchExceptions: true, pythonPath: options.pythonPath, resource: options.resource });
56-
56+
const interpreterService = this.serviceContainer.get<IInterpreterService>(IInterpreterService);
57+
const activatedProcPromise = this.createActivatedEnvironment({ allowEnvironmentFetchExceptions: true, pythonPath: pythonPath, resource: options.resource });
58+
const interpreterInfoPromise = interpreterService.getInterpreterDetails(pythonPath);
5759
// Ensure we do not start multiple daemons for the same interpreter.
5860
// Cache the promise.
5961
const start = async () => {
62+
const interpreter = await interpreterInfoPromise;
6063
const [activatedProc, activatedEnvVars] = await Promise.all([
6164
activatedProcPromise,
62-
this.activationHelper.getActivatedEnvironmentVariables(options.resource, undefined, false)
65+
this.activationHelper.getActivatedEnvironmentVariables(options.resource, interpreter, true)
6366
]);
6467

6568
const daemon = new PythonDaemonExecutionServicePool({...options, pythonPath}, activatedProc!, activatedEnvVars);
Collapse file

‎src/client/datascience/activation.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/activation.ts
+6-9Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,32 @@
44
'use strict';
55

66
import { inject, injectable } from 'inversify';
7-
import { TextDocument } from 'vscode';
87
import { IExtensionSingleActivationService } from '../activation/types';
9-
import { IDocumentManager } from '../common/application/types';
108
import '../common/extensions';
119
import { IPythonExecutionFactory } from '../common/process/types';
1210
import { IDisposableRegistry } from '../common/types';
1311
import { debounceAsync, swallowExceptions } from '../common/utils/decorators';
1412
import { IInterpreterService } from '../interpreter/contracts';
1513
import { PythonDaemonModule } from './constants';
14+
import { INotebookEditor, INotebookEditorProvider } from './types';
1615

1716
@injectable()
1817
export class Activation implements IExtensionSingleActivationService {
1918
private notebookOpened = false;
2019
constructor(
21-
@inject(IDocumentManager) private readonly documentManager: IDocumentManager,
20+
@inject(INotebookEditorProvider) private readonly notebookProvider: INotebookEditorProvider,
2221
@inject(IInterpreterService) private readonly interpreterService: IInterpreterService,
2322
@inject(IPythonExecutionFactory) private readonly factory: IPythonExecutionFactory,
2423
@inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry
2524
) {}
2625
public async activate(): Promise<void> {
27-
this.disposables.push(this.documentManager.onDidOpenTextDocument(this.onDidOpenTextDocument, this));
26+
this.disposables.push(this.notebookProvider.onDidOpenNotebookEditor(this.onDidOpenNotebookEditor, this));
2827
this.disposables.push(this.interpreterService.onDidChangeInterpreter(this.onDidChangeInterpreter, this));
2928
}
3029

31-
private onDidOpenTextDocument(e: TextDocument) {
32-
if (e.fileName.toLowerCase().endsWith('.ipynb')) {
33-
this.notebookOpened = true;
34-
this.PreWarmDaemonPool().ignoreErrors();
35-
}
30+
private onDidOpenNotebookEditor(_: INotebookEditor) {
31+
this.notebookOpened = true;
32+
this.PreWarmDaemonPool().ignoreErrors();
3633
}
3734

3835
private onDidChangeInterpreter() {
Collapse file

‎src/client/datascience/interactive-ipynb/nativeEditorProvider.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/interactive-ipynb/nativeEditorProvider.ts
+6-1Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
'use strict';
44
import { inject, injectable } from 'inversify';
55
import * as path from 'path';
6-
import { TextDocument, TextEditor, Uri } from 'vscode';
6+
import { Event, EventEmitter, TextDocument, TextEditor, Uri } from 'vscode';
77

88
import { ICommandManager, IDocumentManager, IWorkspaceService } from '../../common/application/types';
99
import { JUPYTER_LANGUAGE } from '../../common/constants';
@@ -20,9 +20,13 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp
2020

2121
private activeEditors: Map<string, INotebookEditor> = new Map<string, INotebookEditor>();
2222
private executedEditors: Set<string> = new Set<string>();
23+
private _onDidOpenNotebookEditor = new EventEmitter<INotebookEditor>();
2324
private notebookCount: number = 0;
2425
private openedNotebookCount: number = 0;
2526
private nextNumber: number = 1;
27+
public get onDidOpenNotebookEditor(): Event<INotebookEditor> {
28+
return this._onDidOpenNotebookEditor.event;
29+
}
2630
constructor(
2731
@inject(IServiceContainer) private serviceContainer: IServiceContainer,
2832
@inject(IAsyncDisposableRegistry) asyncRegistry: IAsyncDisposableRegistry,
@@ -170,6 +174,7 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp
170174
this.activeEditors.set(e.file.fsPath, e);
171175
this.disposables.push(e.saved(this.onSavedEditor.bind(this, e.file.fsPath)));
172176
this.openedNotebookCount += 1;
177+
this._onDidOpenNotebookEditor.fire(e);
173178
}
174179

175180
private onSavedEditor(oldPath: string, e: INotebookEditor) {
Collapse file

‎src/client/datascience/jupyter/jupyterCommand.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/jupyter/jupyterCommand.ts
+29-14Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
} from '../../common/process/types';
1515
import { IEnvironmentActivationService } from '../../interpreter/activation/types';
1616
import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts';
17-
import { JupyterCommands } from '../constants';
17+
import { JupyterCommands, PythonDaemonModule } from '../constants';
1818
import { IJupyterCommand, IJupyterCommandFactory } from '../types';
1919

2020
// JupyterCommand objects represent some process that can be launched that should be guaranteed to work because it
@@ -68,9 +68,18 @@ class InterpreterJupyterCommand implements IJupyterCommand {
6868
protected interpreterPromise: Promise<PythonInterpreter | undefined>;
6969
private pythonLauncher: Promise<IPythonExecutionService>;
7070

71-
constructor(protected readonly moduleName: string, protected args: string[], pythonExecutionFactory: IPythonExecutionFactory, private readonly _interpreter: PythonInterpreter) {
71+
constructor(protected readonly moduleName: string, protected args: string[], protected readonly pythonExecutionFactory: IPythonExecutionFactory,
72+
private readonly _interpreter: PythonInterpreter, isActiveInterpreter: boolean) {
7273
this.interpreterPromise = Promise.resolve(this._interpreter);
73-
this.pythonLauncher = pythonExecutionFactory.createActivatedEnvironment({ resource: undefined, interpreter: _interpreter, allowEnvironmentFetchExceptions: true });
74+
this.pythonLauncher = this.interpreterPromise.then(interpreter => {
75+
// Create a daemon only if the interpreter is the same as the current interpreter.
76+
// We don't want too many daemons (one for each of the users interpreter on their machine).
77+
if (isActiveInterpreter) {
78+
return pythonExecutionFactory.createDaemon({ daemonModule: PythonDaemonModule, pythonPath: interpreter!.path });
79+
} else {
80+
return pythonExecutionFactory.createActivatedEnvironment({interpreter: this._interpreter});
81+
}
82+
});
7483
}
7584
public interpreter() : Promise<PythonInterpreter | undefined> {
7685
return this.interpreterPromise;
@@ -80,14 +89,20 @@ class InterpreterJupyterCommand implements IJupyterCommand {
8089
const newOptions = { ...options };
8190
const launcher = await this.pythonLauncher;
8291
const newArgs = [...this.args, ...args];
83-
return launcher.execObservable(newArgs, newOptions);
92+
const moduleName = newArgs[1];
93+
newArgs.shift(); // Remove '-m'
94+
newArgs.shift(); // Remove module name
95+
return launcher.execModuleObservable(moduleName, newArgs, newOptions);
8496
}
8597

8698
public async exec(args: string[], options: SpawnOptions): Promise<ExecutionResult<string>> {
8799
const newOptions = { ...options };
88100
const launcher = await this.pythonLauncher;
89101
const newArgs = [...this.args, ...args];
90-
return launcher.exec(newArgs, newOptions);
102+
const moduleName = newArgs[1];
103+
newArgs.shift(); // Remove '-m'
104+
newArgs.shift(); // Remove module name
105+
return launcher.execModule(moduleName, newArgs, newOptions);
91106
}
92107
}
93108

@@ -99,8 +114,8 @@ class InterpreterJupyterCommand implements IJupyterCommand {
99114
* @implements {IJupyterCommand}
100115
*/
101116
class InterpreterJupyterNotebookCommand extends InterpreterJupyterCommand {
102-
constructor(moduleName: string, args: string[], pythonExecutionFactory: IPythonExecutionFactory, interpreter: PythonInterpreter) {
103-
super(moduleName, args, pythonExecutionFactory, interpreter);
117+
constructor(moduleName: string, args: string[], pythonExecutionFactory: IPythonExecutionFactory, interpreter: PythonInterpreter, isActiveInterpreter: boolean) {
118+
super(moduleName, args, pythonExecutionFactory, interpreter, isActiveInterpreter);
104119
}
105120
}
106121

@@ -109,19 +124,19 @@ class InterpreterJupyterNotebookCommand extends InterpreterJupyterCommand {
109124
export class JupyterCommandFactory implements IJupyterCommandFactory {
110125

111126
constructor(
112-
@inject(IPythonExecutionFactory) private executionFactory: IPythonExecutionFactory,
113-
@inject(IEnvironmentActivationService) private activationHelper : IEnvironmentActivationService,
114-
@inject(IProcessServiceFactory) private processServiceFactory: IProcessServiceFactory,
115-
@inject(IInterpreterService) private interpreterService: IInterpreterService
127+
@inject(IPythonExecutionFactory) private readonly executionFactory : IPythonExecutionFactory,
128+
@inject(IEnvironmentActivationService) private readonly activationHelper : IEnvironmentActivationService,
129+
@inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory,
130+
@inject(IInterpreterService) private readonly interpreterService: IInterpreterService
116131
) {
117132

118133
}
119134

120-
public createInterpreterCommand(command: JupyterCommands, moduleName: string, args: string[], interpreter: PythonInterpreter): IJupyterCommand {
135+
public createInterpreterCommand(command: JupyterCommands, moduleName: string, args: string[], interpreter: PythonInterpreter, isActiveInterpreter: boolean): IJupyterCommand {
121136
if (command === JupyterCommands.NotebookCommand){
122-
return new InterpreterJupyterNotebookCommand(moduleName, args, this.executionFactory, interpreter);
137+
return new InterpreterJupyterNotebookCommand(moduleName, args, this.executionFactory, interpreter, isActiveInterpreter);
123138
}
124-
return new InterpreterJupyterCommand(moduleName, args, this.executionFactory, interpreter);
139+
return new InterpreterJupyterCommand(moduleName, args, this.executionFactory, interpreter, isActiveInterpreter);
125140
}
126141

127142
public createProcessCommand(exe: string, args: string[]): IJupyterCommand {
Collapse file

‎src/client/datascience/jupyter/jupyterCommandFinder.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/jupyter/jupyterCommandFinder.ts
+24-6Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@ import { IApplicationShell, IWorkspaceService } from '../../common/application/t
1010
import { Cancellation, createPromiseFromCancellation, wrapCancellationTokens } from '../../common/cancellation';
1111
import { traceError, traceInfo, traceWarning } from '../../common/logger';
1212
import { IFileSystem } from '../../common/platform/types';
13-
import { IProcessService, IProcessServiceFactory, IPythonExecutionFactory, SpawnOptions } from '../../common/process/types';
13+
import { IProcessService, IProcessServiceFactory, IPythonExecutionFactory, IPythonExecutionService, SpawnOptions } from '../../common/process/types';
1414
import { IConfigurationService, IDisposableRegistry, ILogger, IPersistentState, IPersistentStateFactory } from '../../common/types';
1515
import * as localize from '../../common/utils/localize';
1616
import { StopWatch } from '../../common/utils/stopWatch';
1717
import { IInterpreterService, IKnownSearchPathsForInterpreters, PythonInterpreter } from '../../interpreter/contracts';
1818
import { sendTelemetryEvent } from '../../telemetry';
19-
import { JupyterCommands, RegExpValues, Telemetry } from '../constants';
19+
import { JupyterCommands, PythonDaemonModule, RegExpValues, Telemetry } from '../constants';
2020
import { IJupyterCommand, IJupyterCommandFactory } from '../types';
2121

2222
export enum ModuleExistsStatus {
@@ -141,11 +141,13 @@ export class JupyterCommandFinderImpl {
141141

142142
// If the module is found on this interpreter, then we found it.
143143
if (interpreter && !Cancellation.isCanceled(cancelToken)) {
144-
findResult = await this.doesModuleExist(command, interpreter, cancelToken);
144+
const [result, activeInterpreter] = await Promise.all([this.doesModuleExist(command, interpreter, cancelToken), this.interpreterService.getActiveInterpreter(undefined)]);
145+
findResult = result!;
146+
const isActiveInterpreter = activeInterpreter ? activeInterpreter.path === interpreter.path : false;
145147
if (findResult.status === ModuleExistsStatus.FoundJupyter) {
146-
findResult.command = this.commandFactory.createInterpreterCommand(command, 'jupyter', ['-m', 'jupyter', command], interpreter);
148+
findResult.command = this.commandFactory.createInterpreterCommand(command, 'jupyter', ['-m', 'jupyter', command], interpreter, isActiveInterpreter);
147149
} else if (findResult.status === ModuleExistsStatus.Found) {
148-
findResult.command = this.commandFactory.createInterpreterCommand(command, command, ['-m', command], interpreter);
150+
findResult.command = this.commandFactory.createInterpreterCommand(command, command, ['-m', command], interpreter, isActiveInterpreter);
149151
}
150152
}
151153
return findResult;
@@ -350,13 +352,29 @@ export class JupyterCommandFinderImpl {
350352

351353
return found;
352354
}
355+
private async createExecutionService(interpreter: PythonInterpreter): Promise<IPythonExecutionService> {
356+
const [currentInterpreter, pythonService] = await Promise.all([
357+
this.interpreterService.getActiveInterpreter(undefined),
358+
this.executionFactory.createActivatedEnvironment({ resource: undefined, interpreter, allowEnvironmentFetchExceptions: true })
359+
]);
360+
361+
// Use daemons for current interpreter, when using any other interpreter, do not use a daemon.
362+
// Creating daemons for other interpreters might not be what we want.
363+
// E.g. users can have dozens of pipenv or conda environments.
364+
// In such cases, we'd end up creating n*3 python processes that are long lived.
365+
if (!currentInterpreter || currentInterpreter.path !== interpreter.path){
366+
return pythonService!;
367+
}
368+
369+
return this.executionFactory.createDaemon({ daemonModule: PythonDaemonModule, pythonPath: interpreter.path });
370+
}
353371
private async doesModuleExist(moduleName: string, interpreter: PythonInterpreter, cancelToken?: CancellationToken): Promise<IModuleExistsResult> {
354372
const result: IModuleExistsResult = {
355373
status: ModuleExistsStatus.NotFound
356374
};
357375
if (interpreter && interpreter !== null) {
358376
const newOptions: SpawnOptions = { throwOnStdErr: false, encoding: 'utf8', token: cancelToken };
359-
const pythonService = await this.executionFactory.createActivatedEnvironment({ resource: undefined, interpreter, allowEnvironmentFetchExceptions: true });
377+
const pythonService = await this.createExecutionService(interpreter);
360378

361379
// For commands not 'ipykernel' first try them as jupyter commands
362380
if (moduleName !== JupyterCommands.KernelCreateCommand) {
Collapse file

‎src/client/datascience/jupyter/jupyterExecution.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/jupyter/jupyterExecution.ts
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ export class JupyterExecutionBase implements IJupyterExecution {
5959
this.commandFinder = serviceContainer.get<JupyterCommandFinder>(JupyterCommandFinder);
6060
this.kernelService = new KernelService(this, this.commandFinder, asyncRegistry,
6161
processServiceFactory, interpreterService, fileSystem);
62-
this.notebookStarter = new NotebookStarter(executionFactory, this, this.commandFinder,
62+
this.notebookStarter = new NotebookStarter(executionFactory, this.commandFinder,
6363
this.kernelService, fileSystem, serviceContainer);
6464
this.disposableRegistry.push(this.interpreterService.onDidChangeInterpreter(() => this.onSettingsChanged()));
6565
this.disposableRegistry.push(this);
Collapse file

‎src/client/datascience/jupyter/jupyterSession.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/jupyter/jupyterSession.ts
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ export class JupyterSession implements IJupyterSession {
174174

175175
private async waitForIdleOnSession(session: Session.ISession | undefined, timeout: number): Promise<void> {
176176
if (session && session.kernel) {
177-
traceInfo(`Waiting for idle on: ${session.kernel.id} -> ${session.kernel.status}`);
177+
traceInfo(`Waiting for idle on (kernel): ${session.kernel.id} -> ${session.kernel.status}`);
178178

179179
const statusChangedPromise = new Promise(resolve => session.kernelChanged.connect((_, e) => e.newValue && e.newValue.status === 'idle' ? resolve() : undefined));
180180
const checkStatusPromise = new Promise(async resolve => {
@@ -191,7 +191,7 @@ export class JupyterSession implements IJupyterSession {
191191
resolve();
192192
});
193193
await Promise.race([statusChangedPromise, checkStatusPromise]);
194-
traceInfo(`Finished waiting for idle on: ${session.kernel.id} -> ${session.kernel.status}`);
194+
traceInfo(`Finished waiting for idle on (kernel): ${session.kernel.id} -> ${session.kernel.status}`);
195195

196196
// If we didn't make it out in ten seconds, indicate an error
197197
if (session.kernel && session.kernel.status === 'idle') {
Collapse file

‎src/client/datascience/jupyter/liveshare/hostJupyterServer.ts‎

Copy file name to clipboardExpand all lines: src/client/datascience/jupyter/liveshare/hostJupyterServer.ts
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ export class HostJupyterServer
178178
this.workspaceService);
179179

180180
// Wait for it to be ready
181-
traceInfo(`Waiting for idle ${this.id}`);
181+
traceInfo(`Waiting for idle (session) ${this.id}`);
182182
const stopWatch = new StopWatch();
183183
const idleTimeout = configService.getSettings().datascience.jupyterLaunchTimeout;
184184
await notebook.waitForIdle(idleTimeout);

0 commit comments

Comments
0 (0)
You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.