GeezerVison is showing github.com — No alloc Unicode version by joshuamegnauth54 · Pull Request #9820 · RustPython/RustPython
No alloc Unicode version
#9820
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.rs
(4)
All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
No alloc Unicode version
Simple patch that moves the NameAliases version parser to build.rs. This
avoids embedding the text file just to parse the file line each time
unicode_version() is called.
I also fixed version_owner to allocate less by not constructing a
temporary String to turn into a CString.
Loading branch information
Uh oh!
There was an error while loading. Please reload this page.
commit 419664d6578585839999da13b7629dca3f6514e9
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -31,6 +31,9 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeFormArg {
#[pymodule]
mod unicodedata {
use alloc::ffi::CString;
use rustpython_unicode::{UNICODE_VERSION, UNICODE_VERSION_LEGACY};
use super::{NormalizeFormArg, unicode_core};
use crate::vm::{
Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
Expand Down
Expand Up
@@ -312,23 +315,25 @@ mod unicodedata {
inner: unicode_core::Ucd,
/// Owns the bytes `unidata_version` points at.
#[expect(dead_code, reason = "keeps the version string alive")]
version_owner: alloc::ffi:: CString,
version_owner: CString,
#[pymember]
unidata_version: crate::vm::builtins::descriptor::CStrMember,
}
impl Ucd {
pub(super) fn new(modern: bool) -> Self {
let inner = unicode_core::Ucd::new(modern);
let text = if modern {
unicode_core::unicode_version()
UNICODE_VERSION
} else {
"3.2.0".to_owned()
UNICODE_VERSION_LEGACY
};
let version_owner = alloc::ffi::CString::new(text).unwrap_or_default();
let version_owner = CString::new(text)
.expect("Embedded Unicode version string should never have internal NULs");
let unidata_version =
crate::vm::builtins::descriptor::CStrMember::new(version_owner.as_ptr());
Self {
inner: unicode_core::Ucd::new(modern) ,
inner,
version_owner,
unidata_version,
}
Expand Down
Expand Up
@@ -503,6 +508,6 @@ mod unicodedata {
#[pyattr]
fn unidata_version(_vm: &VirtualMachine) -> String {
unicode_core::unicode_version ()
UNICODE_VERSION.into ()
}
}
Original file line number
Diff line number
Diff line change
Expand Up
@@ -457,17 +457,6 @@ fn generate_algo_names(names: Vec<(u32, Box<str>)>) {
);
}
/// Block names and ranges from the same Unicode release as the other latest tables.
fn generate_blocks() {
let mut writer = open_writer("blocks.rs");
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
}
write_slice_post(&mut writer);
}
/// Formal name aliases (`NameAliases.txt`) and named sequences
/// (`NamedSequences.txt`). `unicodedata.lookup` consults both; `\N{}`
/// accepts only a single character, so named sequences stay out of
Expand Down
Expand Up
@@ -1024,6 +1013,42 @@ fn generate_numeric_type_3_2(rx: mpsc::Receiver<NumericTypeDiffs>) {
);
}
/// Block names and ranges from the same Unicode release as the other latest tables.
fn generate_blocks() {
let mut writer = open_writer("blocks.rs");
write_slice_pre(&mut writer, "BLOCKS", "(u32, u32, &str)");
for line in UnicodeLineReader::from_file_name("Blocks.txt", true) {
let name = line.field(NonZeroUsize::new(1).unwrap(), None);
writeln!(writer, "({}, {}, {name:?}),", line.start, line.end).unwrap();
}
write_slice_post(&mut writer);
}
/// The version string of the latest Unicode database bundled with the standard
/// library (`unicodedata.unidata_version`).
pub fn unicode_version() {
// Report the vendored database version, not the compiler's Unicode version.
let version = open_reader("NameAliases.txt", true)
.lines()
.next()
.expect("should be able to read from vendored NameAliases.txt")
.expect("vendored NameAliases.txt must start with its Unicode version");
let version = version
.strip_prefix("# NameAliases-")
.expect("NameAliases.txt doesn't start with version info")
.strip_suffix(".txt")
.unwrap()
.trim();
println!("cargo:rustc-env=UNICODE_VERSION={version}");
}
/// Parsers that are standalone and read small files.
fn small_parsers() {
generate_blocks();
unicode_version();
}
/// Run each UCD parser in parallel.
fn drive_parsers() {
let parsers = [
Expand All
@@ -1034,7 +1059,7 @@ fn drive_parsers() {
binary_props_3_2,
combining_class_3_2,
generate_name_lookups,
generate_blocks ,
small_parsers ,
];
let mut handles = Vec::with_capacity(parsers.len() + 2);
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -19,6 +19,9 @@ use icu_properties::props::{
};
use rustpython_wtf8::CodePoint;
pub const UNICODE_VERSION: &str = env!("UNICODE_VERSION");
pub const UNICODE_VERSION_LEGACY: &str = "3.2.0";
// Latest Unicode
include!(concat!(env!("OUT_DIR"), "/generated/blocks.rs"));
include!(concat!(env!("OUT_DIR"), "/generated/algo_names.rs"));
Expand Down
Expand Up
@@ -237,20 +240,6 @@ pub fn membership_3_2(ch: u32) -> bool {
.is_ok()
}
/// The version string of the latest Unicode database bundled with the standard
/// library (`unicodedata.unidata_version`).
#[must_use]
pub fn unicode_version() -> String {
// Report the vendored database version, not the compiler's Unicode version.
include_str!("../unicode/latest/NameAliases.txt")
.lines()
.next()
.and_then(|header| header.strip_prefix("# NameAliases-"))
.and_then(|header| header.strip_suffix(".txt"))
.expect("vendored NameAliases.txt must start with its Unicode version")
.into()
}
/// Look up a name in the latest UCD (`unicodedata.lookup`).
///
/// Named sequences are included; `\N{...}` should use [`lookup_character`]
Expand Down
Expand Up
@@ -546,9 +535,9 @@ impl Ucd {
#[must_use]
pub fn unidata_version(&self) -> String {
if self.modern {
unicode_version ()
UNICODE_VERSION.into ()
} else {
"3.2.0" .into()
UNICODE_VERSION_LEGACY .into()
}
}
}
Expand All
@@ -565,7 +554,7 @@ mod tests {
#[test]
fn version_matches_vendored_database() {
assert_eq!(super::unicode_version() , "17.0.0");
assert_eq!(super::UNICODE_VERSION , "17.0.0");
assert_eq!(Ucd::new(true).unidata_version(), "17.0.0");
}
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -16,5 +16,8 @@ pub mod grapheme;
pub mod identifier;
pub mod normalize;
pub use data::{LookupResult, Ucd, character_name, lookup_character, lookup_name, unicode_version};
pub use data::{
LookupResult, UNICODE_VERSION, UNICODE_VERSION_LEGACY, Ucd, character_name, lookup_character,
lookup_name,
};
pub use normalize::{NormalizeForm, is_normalized, normalize};
Loading
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.