GeezerVison is showing github.com — stringstream-lua/stringstream.lua at main · gilzoide/stringstream-lua

Expand file tree

stringstream.lua

Copy path

More file actions

Latest commit

History

426 lines (394 loc) · 15.3 KB

stringstream.lua

Copy path

File metadata and controls

426 lines (394 loc) · 15.3 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

--- An object that loads chunks of strings on demand compatible with a subset

-- of the string API suitable for parsing.

-- Project URL: https://github.com/gilzoide/stringstream-lua

-- Example:

-- local stringstream = require 'stringstream'

-- -- Streams may be created with callable objects (functions or tables/userdata

-- -- with __call metamethod) like the ones `load` expects, or file-like objects

-- -- that contain a `read` method, like open files.

-- local stream = assert(stringstream.new(io.stdin))

-- -- Alternatively, `stringstream.open(filename, ...)` may be used to open a file

-- -- by name in read mode and create a stringstream from it.

-- --

-- -- local stream = assert(stringstream.open("README.md"))

-- -- Now just call the supported string methods =D

-- while true do

-- local token, advance = stream:match("(%S+)()")

-- if not token then break end

-- -- ... do something with token

-- print('TOKEN', token)

-- stream = stream:sub(advance)

-- end

-- @module stringstream

-- @license [Public Domain](https://unlicense.org/)

-- Lua 5.1+ compatibility

local unpack = table.unpack or unpack

--- Internal Stream state, caching chunks

local stream_state = {}

stream_state.__index = stream_state

function stream_state.new(read, ...)

local state = setmetatable({

read = read,

contents = {},

first_loaded_chunk = 1,

last_loaded_chunk = 0,

loaded_length = 0,

read_args = { ... },

}, stream_state)

state:load_next()

return state

end

function stream_state:calculate_chunk_index(from_chunk, starting_index)

-- @warning: only call after checking `load_if_needed`

local chunk_start = from_chunk

local chunk = self.contents[chunk_start]

while starting_index > #chunk do

chunk_start = chunk_start + 1

starting_index = starting_index - #chunk

chunk = self.contents[chunk_start]

end

return chunk_start, starting_index

end

function stream_state:load_if_needed(from_chunk, starting_index)

assert(from_chunk >= self.first_loaded_chunk, "Seeking stream backwards is not supported")

-- TODO: after thorough testing, remove this next assertion

assert(from_chunk <= self.last_loaded_chunk, "Invalid chunk number for stream, not loaded yet")

-- unload chunks that are assumed to not be needed anymore

self:unload_until(from_chunk)

-- check how long loaded content is, to check if loading is necessary

local loaded_length = self.loaded_length

for i = self.first_loaded_chunk, from_chunk - 1 do

loaded_length = loaded_length - #self.contents[i]

end

-- loads contents until `starting_index` is valid, returns `false` if not enough content is available

while loaded_length < starting_index do

local new_content = self:load_next()

if not new_content then

return false

end

loaded_length = loaded_length + #new_content

end

return loaded_length - (starting_index - 1)

end

function stream_state:load_next()

local new_content = self.read(unpack(self.read_args))

--print(' !!! loading', self.last_loaded_chunk + 1, string.format("%q", new_content))

if not new_content or new_content == '' then

return nil

end

self.last_loaded_chunk = self.last_loaded_chunk + 1

self.contents[self.last_loaded_chunk] = new_content

self.loaded_length = self.loaded_length + #new_content

return new_content

end

function stream_state:unload_until(chunk)

for i = self.first_loaded_chunk, chunk - 2 do

--print(' !!! unloading', i)

self.loaded_length = self.loaded_length - #self.contents[i]

self.contents[i] = nil

self.first_loaded_chunk = i + 1

end

function stream_state:string_from(chunk, starting_index, end_index)

assert(chunk >= self.first_loaded_chunk, "Trying to access unloaded chunks")

local content = table.concat(self.contents, '', chunk, self.last_loaded_chunk)

--print(' !!! STRING', chunk, starting_index, end_index, self.contents[chunk] ~= nil, self.contents[self.last_loaded_chunk] ~= nil)

return content:sub(starting_index, end_index)

end

function stream_state:__len()

return self.loaded_length

end

function stream_state:__tostring()

return self:string_from(self.first_loaded_chunk, 1)

end

--- String Stream module

local stringstream = {}

--- Default read size of streams created with file objects.

-- Change this to configure defaults module-wise.

-- Default value is 1024.

stringstream.readsize = 1024

--- Default maximum number of bytes that `stringstream:find` may load.

-- Change this to configure defaults module-wise.

-- Default value is `math.huge`, that is, load until stream is completely consumed, if necessary.

-- @see stringstream:find

stringstream.max_find_lookahead = math.huge

local function iscallable(v)

if type(v) == 'function' then

return true

else

local mt = getmetatable(v)

return mt and mt.__call

end

local function wrap_stream(stream, max_find_lookahead, chunk, starting_index)

return setmetatable({

stream = stream,

max_find_lookahead = max_find_lookahead or stringstream.max_find_lookahead,

chunk = chunk or stream.first_loaded_chunk,

starting_index = starting_index or 1,

}, stringstream)

end

--- Create a new stringstream from callable object or file-like object.

-- Both functions and tables or userdata with a `__call` metamethod are

-- accepted as callables. Just like `load`, each call must return a string that

-- concatenates with previous results and a return of an empty string or a falsey

-- value signals the end of the stream.

-- File-like objects are tables or userdata with a `read` method.

-- Chunks will be read calling the `read` method as in `object:read(...)`.

-- @usage

-- -- stream that reads chunks of 1024 bytes from opened file

-- -- and retries `find` as many times as necessary

-- stream = stringstream.new(io.open('file'))

-- -- stream that reads lines from stdin and fails a search after

-- -- having more than 4096 bytes loaded

-- stream = stringstream.new(io.stdin, { max_find_lookahead = 4096 }, '*l')

-- -- stream that generates digits from 1 to 10

-- local gen_numbers = coroutine.wrap(function()

-- for i = 1, 10 do

-- coroutine.yield(tostring(i))

-- end

-- end)

-- stream = stringstream.new(gen_numbers)

-- @tparam ?function|table|userdata callable_or_file Callable or file-like object which chunks will be read from

-- @param[opt] options Table of stream options. Currently the only supported option is `max_find_lookahead`.

-- @param[opt] ... Arguments to be forwarded to the reading function. If `callable_or_file` is a file and no extra arguments are passed, `stringstream.readsize` will be used

-- @return[1] stringstream object

-- @return[2] nil

-- @return[2] error message

function stringstream.new(callable_or_file, options, ...)

local read, should_forward_default_chunksize

if iscallable(callable_or_file) then

read = callable_or_file

else

local read_type = type(callable_or_file)

if read_type == 'table' or read_type == 'userdata' then

if not callable_or_file.read then

return nil, 'Argument is not a file-like object, no "read" method found'

end

should_forward_default_chunksize = select('#', ...) == 0 and io.type(callable_or_file) == 'file'

read = function(...) return callable_or_file:read(...) end

else

return nil, string.format("Expected callable or file-like object, found %q", read_type)

end

local stream

if should_forward_default_chunksize then

stream = stream_state.new(read, stringstream.readsize)

else

stream = stream_state.new(read, ...)

end

return wrap_stream(stream, options and options.max_find_lookahead, 1, 1)

end

--- Creates a new stream from file.

-- Opens the file with `io.open` in read mode and forward parameters to

-- `stringstream.new`.

-- @see stringstream.new

-- @usage

-- -- stream that reads chunks of 1024 bytes from 'file.txt'

-- -- and loads the entire contents on a call to `find`, if necessary

-- stream = stringstream.open('file.txt')

-- -- stream that reads lines from 'file.txt'

-- -- and loads the entire contents on a call to `find`, if necessary

-- stream = stringstream.open('file.txt', nil, '*l')

-- @tparam string filename Name of the file to open

-- @param[opt] ... Arguments forwarded to `stringstream:new`

-- @return[1] stringstream object

-- @treturn[2] nil

-- @treturn[2] string Error message

-- @treturn[2] number Error code on `io.open` failure

function stringstream.open(filename, ...)

local file, err, code = io.open(filename, 'r')

if not file then

return nil, err, code

end

return stringstream.new(file, ...)

end

--- Returns a substring or a new view into stream.

-- If both `i` and `j` are passed, returns the substring that starts at `i`

-- and continues until `j` or until the end of stream.

-- Otherwise, returns a new stringstream object with starting index `i`.

-- @raise If either `i` or `j` are non-positive, as reading from the end of

-- stream is not supported.

-- @tparam number i Starting index

-- @tparam[opt] number j Starting index

-- @treturn[1] stringstream If `j` is not passed and `i` is not past the end of the stream

-- @treturn[2] string Empty if `j` is not passed in and `i` is past the end of the stream

-- @treturn[3] string Substring if `j` is passed in

function stringstream:sub(i, j)

assert(i and i > 0, "Calling stringstream.sub with non-positive index is not supported!")

if not j then

local starting_index = self.starting_index + i - 1

if self.stream:load_if_needed(self.chunk, starting_index) then

return wrap_stream(self.stream, self.max_find_lookahead, self.stream:calculate_chunk_index(self.chunk, starting_index))

else

return ''

end

else

assert(j > 0, "Calling stringstream.sub with negative index is not supported!")

if j < i then

return ''

end

local starting_index = self.starting_index + i - 1

local ending_index = self.starting_index + j - 1

self.stream:load_if_needed(self.chunk, ending_index)

return self.stream:string_from(self.chunk, starting_index, ending_index)

end

--- Try finding `pattern` on loaded contents.

-- Upon failure or if match spans the entire loaded content, loads new chunks,

-- bailing out after having more than `max_find_lookahead` bytes loaded

-- counting from `init`, returning `nil` afterwards.

-- Notice that the default value for `max_find_lookahead` makes the whole stream be

-- loaded in case of failures or big matches. This is to have consistent output

-- between stringstream and the string API by default.

-- @raise If `init` is a negative number.

-- @see string:find

-- @tparam string pattern Pattern string to search

-- @tparam[opt] number init Where to start the search from

-- @param[opt] plain If truthy, turns off the pattern matching facilities

-- @treturn[1] number Starting index of found pattern

-- @treturn[1] number Ending index of found pattern

-- @treturn[1] ... Aditional captures of found pattern

-- @treturn[2] fail If pattern is not found

function stringstream:find(pattern, init, plain)

init = init or 1

assert(init >= 0, "Calling stringstream.find with negative index is not supported!")

local stream, chunk, starting_index = self.stream, self.chunk, self.starting_index

local loaded_length = stream:load_if_needed(chunk, starting_index + init - 1)

while loaded_length < self.max_find_lookahead do

local text = stream:string_from(chunk, starting_index)

local results = { text:find(pattern, init, plain) }

if results[1] and results[2] < #text then

return unpack(results)

-- TODO: check if `init` may be updated with `results[1]` in case a

-- match was found, but may be longer, to reduce search space

-- (check if there are cases where this would return wrong results)

end

local extra_contents = stream:load_next()

if not extra_contents then

return unpack(results)

end

loaded_length = loaded_length + #extra_contents

end

return nil

end

--- Try matching `pattern` on loaded contents.

-- Uses `stringstream:find` for searching, so the same caveats apply.

-- @raise If `init` is a negative number.

-- @see string:match

-- @see stringstream:find

-- @tparam string pattern Pattern string to search

-- @tparam[opt] number init Where to start the search from

-- @treturn[1] ... Captures of the found pattern

-- @treturn[2] string The whole match, if pattern specifies no captures

-- @treturn[3] fail If pattern is not found

function stringstream:match(pattern, init)

assert((not init) or init >= 0, "Calling stringstream.match with negative index is not supported!")

local find_results = { self:find(pattern, init, false) }

if find_results[1] and not find_results[3] then

return self:sub(find_results[1], find_results[2])

else

return unpack(find_results, 3)

end

--- Returns an iterator function that, each time it is called, returns the next

-- match from loaded contents.

-- Uses `stringstream:find` for searching, so the same caveats apply.

-- @raise If `init` is a negative number.

-- @see string:gmatch

-- @see stringstream:match

-- @tparam string pattern Pattern string to search

-- @tparam[opt] number init Where to start the search from

-- @treturn function Iterator function over matches

function stringstream:gmatch(pattern, init)

assert((not init) or init >= 0, "Calling stringstream.gmatch with negative index is not supported!")

-- Don't interpreted '^' as an anchor, just like `string.gmatch` does

if pattern:sub(1, 1) == '^' then

pattern = '%' .. pattern

end

local function iterate()

local init = init

while true do

local find_results = { self:find(pattern, init, false) }

if find_results[1] then

--print('GMATCH', unpack(find_results))

if not find_results[3] then

coroutine.yield(self:sub(find_results[1], find_results[2]))

else

coroutine.yield(unpack(find_results, 3))

end

init = find_results[2] + 1

else

return nil

end

return coroutine.wrap(iterate)

end

--- Returns the current loaded content string.

-- Be careful that it almost never reflects the entire content string.

function stringstream:__tostring()

return self.stream:string_from(self.chunk, self.starting_index)

end

--- Returns the current loaded content length.

-- Be careful that it almost never reflects the entire content length.

function stringstream:__len()

return self.stream.loaded_length - (self.starting_index - 1)

end

--- Alias for `stringstream:__len`

-- @function stringstream.len

stringstream.__index = {

sub = stringstream.sub,

find = stringstream.find,

match = stringstream.match,

gmatch = stringstream.gmatch,

len = stringstream.__len,

__len = stringstream.__len,

}

--- Module version.

stringstream._VERSION = '0.2.0'

return stringstream