From edccd966433e0ca0460a2e05e42150e34f815adb Mon Sep 17 00:00:00 2001 From: Eduard Filip Date: Wed, 22 Jan 2025 10:48:24 +0100 Subject: [PATCH] Add const support for integer types (#218) * Add Rust constant type * Add constant parsing functionality * Add basic structure for generating consts * Add Go const generation * Add Python const generation * Add Typescript const generation * Add test for generating constants --- core/data/tests/can_generate_const/input.rs | 2 + core/data/tests/can_generate_const/output.go | 5 ++ core/data/tests/can_generate_const/output.py | 6 ++ core/data/tests/can_generate_const/output.ts | 1 + core/src/language/go.rs | 22 +++++- core/src/language/kotlin.rs | 6 +- core/src/language/mod.rs | 16 +++- core/src/language/python.rs | 25 +++++- core/src/language/scala.rs | 10 ++- core/src/language/swift.rs | 8 +- core/src/language/typescript.rs | 22 +++++- core/src/parser.rs | 83 ++++++++++++++++++-- core/src/rust_types.rs | 32 ++++++++ core/src/topsort.rs | 17 +++- core/src/visitors.rs | 23 +++++- core/tests/snapshot_tests.rs | 1 + 16 files changed, 260 insertions(+), 19 deletions(-) create mode 100644 core/data/tests/can_generate_const/input.rs create mode 100644 core/data/tests/can_generate_const/output.go create mode 100644 core/data/tests/can_generate_const/output.py create mode 100644 core/data/tests/can_generate_const/output.ts diff --git a/core/data/tests/can_generate_const/input.rs b/core/data/tests/can_generate_const/input.rs new file mode 100644 index 00000000..f08d4043 --- /dev/null +++ b/core/data/tests/can_generate_const/input.rs @@ -0,0 +1,2 @@ +#[typeshare] +pub const MY_VAR: u32 = 12; diff --git a/core/data/tests/can_generate_const/output.go b/core/data/tests/can_generate_const/output.go new file mode 100644 index 00000000..e8201346 --- /dev/null +++ b/core/data/tests/can_generate_const/output.go @@ -0,0 +1,5 @@ +package proto + +import "encoding/json" + +const MyVar uint32 = 12 diff --git a/core/data/tests/can_generate_const/output.py b/core/data/tests/can_generate_const/output.py new file mode 100644 index 00000000..58be6bec --- /dev/null +++ b/core/data/tests/can_generate_const/output.py @@ -0,0 +1,6 @@ +from __future__ import annotations + + + + +MY_VAR: int = 12 diff --git a/core/data/tests/can_generate_const/output.ts b/core/data/tests/can_generate_const/output.ts new file mode 100644 index 00000000..e742d471 --- /dev/null +++ b/core/data/tests/can_generate_const/output.ts @@ -0,0 +1 @@ +export const MY_VAR: number = 12; diff --git a/core/src/language/go.rs b/core/src/language/go.rs index b67c9a7c..a5d6a6bb 100644 --- a/core/src/language/go.rs +++ b/core/src/language/go.rs @@ -3,7 +3,7 @@ use std::io::Write; use crate::language::SupportedLanguage; use crate::parser::ParsedData; use crate::rename::RenameExt; -use crate::rust_types::{RustItem, RustTypeFormatError, SpecialRustType}; +use crate::rust_types::{RustConst, RustConstExpr, RustItem, RustTypeFormatError, SpecialRustType}; use crate::{ language::Language, rust_types::{RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, @@ -58,6 +58,7 @@ impl Language for Go { structs, enums, aliases, + consts, .. } = data; @@ -66,6 +67,7 @@ impl Language for Go { .map(RustItem::Alias) .chain(structs.into_iter().map(RustItem::Struct)) .chain(enums.into_iter().map(RustItem::Enum)) + .chain(consts.into_iter().map(RustItem::Const)) .collect::>(); topsort(&mut items); @@ -96,6 +98,7 @@ impl Language for Go { RustItem::Enum(e) => self.write_enum(w, e, &types_mapping_to_struct)?, RustItem::Struct(s) => self.write_struct(w, s)?, RustItem::Alias(a) => self.write_type_alias(w, a)?, + RustItem::Const(c) => self.write_const(w, c)?, } } @@ -189,6 +192,23 @@ impl Language for Go { Ok(()) } + fn write_const(&mut self, w: &mut dyn Write, c: &RustConst) -> std::io::Result<()> { + match c.expr { + RustConstExpr::Int(val) => { + let const_type = self + .format_type(&c.r#type, &[]) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + writeln!( + w, + "const {} {} = {}", + c.id.renamed.to_pascal_case(), + const_type, + val + ) + } + } + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { write_comments(w, 0, &rs.comments)?; // TODO: Support generic bounds: https://github.com/1Password/typeshare/issues/222 diff --git a/core/src/language/kotlin.rs b/core/src/language/kotlin.rs index 69ce4d7b..95d09d7a 100644 --- a/core/src/language/kotlin.rs +++ b/core/src/language/kotlin.rs @@ -4,7 +4,7 @@ use crate::parser::{remove_dash_from_identifier, DecoratorKind, ParsedData}; use crate::rust_types::{RustTypeFormatError, SpecialRustType}; use crate::{ rename::RenameExt, - rust_types::{Id, RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, + rust_types::{Id, RustConst, RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, }; use itertools::Itertools; use joinery::JoinableIterator; @@ -174,6 +174,10 @@ impl Language for Kotlin { Ok(()) } + fn write_const(&mut self, _w: &mut dyn Write, _c: &RustConst) -> std::io::Result<()> { + todo!() + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { self.write_comments(w, 0, &rs.comments)?; writeln!(w, "@Serializable")?; diff --git a/core/src/language/mod.rs b/core/src/language/mod.rs index 4eef6da7..4d007411 100644 --- a/core/src/language/mod.rs +++ b/core/src/language/mod.rs @@ -1,7 +1,7 @@ use crate::{ parser::{ParseError, ParsedData}, rust_types::{ - Id, RustEnum, RustEnumVariant, RustItem, RustStruct, RustType, RustTypeAlias, + Id, RustConst, RustEnum, RustEnumVariant, RustItem, RustStruct, RustType, RustTypeAlias, RustTypeFormatError, SpecialRustType, }, topsort::topsort, @@ -173,6 +173,7 @@ pub trait Language { structs, enums, aliases, + consts, .. } = data; @@ -181,7 +182,8 @@ pub trait Language { .into_iter() .map(RustItem::Alias) .chain(structs.into_iter().map(RustItem::Struct)) - .chain(enums.into_iter().map(RustItem::Enum)), + .chain(enums.into_iter().map(RustItem::Enum)) + .chain(consts.into_iter().map(RustItem::Const)), ); topsort(&mut items); @@ -191,6 +193,7 @@ pub trait Language { RustItem::Enum(e) => self.write_enum(writable, e)?, RustItem::Struct(s) => self.write_struct(writable, s)?, RustItem::Alias(a) => self.write_type_alias(writable, a)?, + RustItem::Const(c) => self.write_const(writable, c)?, } } @@ -299,6 +302,15 @@ pub trait Language { Ok(()) } + /// Write a constant variable. + /// Example of a constant variable: + /// ``` + /// const ANSWER_TO_EVERYTHING: u32 = 42; + /// ``` + fn write_const(&mut self, _w: &mut dyn Write, _c: &RustConst) -> std::io::Result<()> { + Ok(()) + } + /// Write a struct by converting it /// Example of a struct: /// ```ignore diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 0e0a9f1a..63573082 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -1,9 +1,12 @@ use crate::parser::ParsedData; use crate::rust_types::{RustEnumShared, RustItem, RustType, RustTypeFormatError, SpecialRustType}; use crate::topsort::topsort; +use crate::RenameExt; use crate::{ language::Language, - rust_types::{RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, + rust_types::{ + RustConst, RustConstExpr, RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias, + }, }; use std::collections::HashSet; use std::hash::Hash; @@ -98,6 +101,7 @@ impl Language for Python { structs, enums, aliases, + consts, .. } = data; @@ -106,6 +110,7 @@ impl Language for Python { .map(RustItem::Alias) .chain(structs.into_iter().map(RustItem::Struct)) .chain(enums.into_iter().map(RustItem::Enum)) + .chain(consts.into_iter().map(RustItem::Const)) .collect::>(); topsort(&mut items); @@ -116,6 +121,7 @@ impl Language for Python { RustItem::Enum(e) => self.write_enum(&mut body, &e)?, RustItem::Struct(rs) => self.write_struct(&mut body, &rs)?, RustItem::Alias(t) => self.write_type_alias(&mut body, &t)?, + RustItem::Const(c) => self.write_const(&mut body, &c)?, }; } @@ -243,6 +249,23 @@ impl Language for Python { Ok(()) } + fn write_const(&mut self, w: &mut dyn Write, c: &RustConst) -> std::io::Result<()> { + match c.expr { + RustConstExpr::Int(val) => { + let const_type = self + .format_type(&c.r#type, &[]) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + writeln!( + w, + "{}: {} = {}", + c.id.renamed.to_snake_case().to_uppercase(), + const_type, + val + ) + } + } + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { { rs.generic_types diff --git a/core/src/language/scala.rs b/core/src/language/scala.rs index 9ba736d7..45bc3888 100644 --- a/core/src/language/scala.rs +++ b/core/src/language/scala.rs @@ -1,8 +1,10 @@ use super::{CrateTypes, Language}; use crate::language::SupportedLanguage; use crate::parser::{remove_dash_from_identifier, ParsedData}; -use crate::rust_types::{RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}; -use crate::rust_types::{RustType, RustTypeFormatError, SpecialRustType}; +use crate::rust_types::{ + RustConst, RustEnum, RustEnumVariant, RustField, RustStruct, RustType, RustTypeAlias, + RustTypeFormatError, SpecialRustType, +}; use itertools::Itertools; use joinery::JoinableIterator; use lazy_format::lazy_format; @@ -150,6 +152,10 @@ impl Language for Scala { Ok(()) } + fn write_const(&mut self, _w: &mut dyn Write, _c: &RustConst) -> std::io::Result<()> { + todo!() + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { self.write_comments(w, 0, &rs.comments)?; diff --git a/core/src/language/swift.rs b/core/src/language/swift.rs index 30c7882b..085305a7 100644 --- a/core/src/language/swift.rs +++ b/core/src/language/swift.rs @@ -3,8 +3,8 @@ use crate::{ parser::{remove_dash_from_identifier, DecoratorKind, ParsedData}, rename::RenameExt, rust_types::{ - DecoratorMap, RustEnum, RustEnumVariant, RustStruct, RustTypeAlias, RustTypeFormatError, - SpecialRustType, + DecoratorMap, RustConst, RustEnum, RustEnumVariant, RustStruct, RustTypeAlias, + RustTypeFormatError, SpecialRustType, }, GenerationError, }; @@ -258,6 +258,10 @@ impl Language for Swift { Ok(()) } + fn write_const(&mut self, _w: &mut dyn Write, _c: &RustConst) -> std::io::Result<()> { + todo!() + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> io::Result<()> { let mut coding_keys = vec![]; let mut should_write_coding_keys = false; diff --git a/core/src/language/typescript.rs b/core/src/language/typescript.rs index 184dc9db..f29e9aed 100644 --- a/core/src/language/typescript.rs +++ b/core/src/language/typescript.rs @@ -1,9 +1,10 @@ +use crate::RenameExt; use crate::{ language::{Language, SupportedLanguage}, parser::ParsedData, rust_types::{ - RustEnum, RustEnumVariant, RustField, RustStruct, RustType, RustTypeAlias, - RustTypeFormatError, SpecialRustType, + RustConst, RustConstExpr, RustEnum, RustEnumVariant, RustField, RustStruct, RustType, + RustTypeAlias, RustTypeFormatError, SpecialRustType, }, }; use itertools::Itertools; @@ -120,6 +121,23 @@ impl Language for TypeScript { Ok(()) } + fn write_const(&mut self, w: &mut dyn Write, c: &RustConst) -> io::Result<()> { + match c.expr { + RustConstExpr::Int(val) => { + let const_type = self + .format_type(&c.r#type, &[]) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + writeln!( + w, + "export const {}: {} = {};", + c.id.renamed.to_snake_case().to_uppercase(), + const_type, + val + ) + } + } + } + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> io::Result<()> { self.write_comments(w, 0, &rs.comments)?; writeln!( diff --git a/core/src/parser.rs b/core/src/parser.rs index e0cfbd2d..53b9ed5d 100644 --- a/core/src/parser.rs +++ b/core/src/parser.rs @@ -3,9 +3,9 @@ use crate::{ language::{CrateName, SupportedLanguage}, rename::RenameExt, rust_types::{ - DecoratorMap, FieldDecorator, Id, RustEnum, RustEnumShared, RustEnumVariant, - RustEnumVariantShared, RustField, RustItem, RustStruct, RustType, RustTypeAlias, - RustTypeParseError, + DecoratorMap, FieldDecorator, Id, RustConst, RustConstExpr, RustEnum, RustEnumShared, + RustEnumVariant, RustEnumVariantShared, RustField, RustItem, RustStruct, RustType, + RustTypeAlias, RustTypeParseError, SpecialRustType, }, target_os_check::accept_target_os, visitors::{ImportedType, TypeShareVisitor}, @@ -20,8 +20,8 @@ use std::{ }; use syn::{ ext::IdentExt, parse::ParseBuffer, punctuated::Punctuated, visit::Visit, Attribute, Expr, - ExprLit, Fields, GenericParam, ItemEnum, ItemStruct, ItemType, LitStr, Meta, MetaList, - MetaNameValue, Token, + ExprLit, Fields, GenericParam, ItemConst, ItemEnum, ItemStruct, ItemType, Lit, LitStr, Meta, + MetaList, MetaNameValue, Token, }; use thiserror::Error; @@ -74,6 +74,10 @@ pub enum ParseError { SerdeTagRequired { enum_ident: String }, #[error("serde content attribute needs to be specified for algebraic enum {enum_ident}. e.g. #[serde(tag = \"type\", content = \"content\")]")] SerdeContentRequired { enum_ident: String }, + #[error("the expression assigned to this constant variable is not a numeric literal")] + RustConstExprInvalid, + #[error("you cannot use typeshare on a constant that is not a numeric literal")] + RustConstTypeInvalid, #[error("the serde flatten attribute is not currently supported")] SerdeFlattenNotAllowed, #[error("IO error: {0}")] @@ -98,6 +102,8 @@ pub struct ParsedData { pub enums: Vec, /// Type aliases defined in the source pub aliases: Vec, + /// Constant variables defined in the source + pub consts: Vec, /// Imports used by this file pub import_types: HashSet, /// Crate this belongs to. @@ -119,6 +125,7 @@ impl AddAssign for ParsedData { self.structs.append(&mut rhs.structs); self.enums.append(&mut rhs.enums); self.aliases.append(&mut rhs.aliases); + self.consts.append(&mut rhs.consts); self.import_types.extend(rhs.import_types); self.type_names.extend(rhs.type_names); self.errors.append(&mut rhs.errors); @@ -154,6 +161,10 @@ impl ParsedData { self.type_names.insert(a.id.renamed.clone()); self.aliases.push(a); } + RustItem::Const(c) => { + self.type_names.insert(c.id.renamed.clone()); + self.consts.push(c); + } } } @@ -162,6 +173,7 @@ impl ParsedData { self.structs.is_empty() && self.enums.is_empty() && self.aliases.is_empty() + && self.consts.is_empty() && self.errors.is_empty() } } @@ -498,6 +510,67 @@ pub(crate) fn parse_type_alias(t: &ItemType) -> Result { })) } +/// Parses a const variant. +pub(crate) fn parse_const(c: &ItemConst) -> Result { + let expr = parse_const_expr(&c.expr)?; + + // serialized_as needs to be supported in case the user wants to use a different type + // for the constant variable in a different language + let ty = if let Some(ty) = get_serialized_as_type(&c.attrs) { + ty.parse()? + } else { + RustType::try_from(c.ty.as_ref())? + }; + + match &ty { + RustType::Special(SpecialRustType::HashMap(_, _)) + | RustType::Special(SpecialRustType::Vec(_)) + | RustType::Special(SpecialRustType::Option(_)) => { + return Err(ParseError::RustConstTypeInvalid); + } + RustType::Special(_) => (), + RustType::Simple { .. } => (), + _ => return Err(ParseError::RustConstTypeInvalid), + }; + + Ok(RustItem::Const(RustConst { + id: get_ident(Some(&c.ident), &c.attrs, &None), + r#type: ty, + expr, + })) +} + +fn parse_const_expr(e: &Expr) -> Result { + struct ExprLitVisitor(pub Option>); + impl Visit<'_> for ExprLitVisitor { + fn visit_expr_lit(&mut self, el: &ExprLit) { + if self.0.is_some() { + // should we throw an error instead of silently ignoring a second literal? + // or would this create false positives? + return; + } + let check_literal_type = || { + Ok(match &el.lit { + Lit::Int(lit_int) => { + let int: i128 = lit_int + .base10_parse() + .map_err(|_| ParseError::RustConstTypeInvalid)?; + RustConstExpr::Int(int) + } + _ => return Err(ParseError::RustConstTypeInvalid), + }) + }; + + self.0.replace(check_literal_type()); + } + } + let mut expr_visitor = ExprLitVisitor(None); + syn::visit::visit_expr(&mut expr_visitor, e); + expr_visitor + .0 + .unwrap_or(Err(ParseError::RustConstTypeInvalid)) +} + // Helpers /// Checks the given attrs for `#[typeshare]` diff --git a/core/src/rust_types.rs b/core/src/rust_types.rs index 13be15c8..cb231384 100644 --- a/core/src/rust_types.rs +++ b/core/src/rust_types.rs @@ -74,6 +74,36 @@ impl Ord for RustStruct { } } +/// Rust const variable. +/// +/// Typeshare can only handle numeric and string constants. +/// ``` +/// pub const MY_CONST: &str = "constant value"; +/// ``` +#[derive(Debug, Clone)] +pub struct RustConst { + /// The identifier for the constant. + pub id: Id, + /// The type identifier that this constant is referring to. + pub r#type: RustType, + /// The expression that the constant contains. + pub expr: RustConstExpr, +} + +impl PartialEq for RustConst { + fn eq(&self, other: &Self) -> bool { + self.id.original == other.id.original + } +} + +/// A constant expression that can be shared via a constant variable across the typeshare +/// boundary. +#[derive(Debug, Clone)] +pub enum RustConstExpr { + /// Expression represents an integer. + Int(i128), +} + /// Rust type alias. /// ``` /// pub struct MasterPassword(String); @@ -698,4 +728,6 @@ pub enum RustItem { Enum(RustEnum), /// A `type` definition or newtype struct. Alias(RustTypeAlias), + /// A `const` definition + Const(RustConst), } diff --git a/core/src/topsort.rs b/core/src/topsort.rs index 20fa034b..fe99b72f 100644 --- a/core/src/topsort.rs +++ b/core/src/topsort.rs @@ -1,7 +1,8 @@ use std::collections::{HashMap, HashSet}; use crate::rust_types::{ - RustEnum, RustEnumVariant, RustItem, RustStruct, RustType, RustTypeAlias, SpecialRustType, + RustConst, RustEnum, RustEnumVariant, RustItem, RustStruct, RustType, RustTypeAlias, + SpecialRustType, }; fn get_dependencies_from_type( @@ -120,6 +121,18 @@ fn get_type_alias_dependencies( } } +fn get_const_dependencies( + c: &RustConst, + types: &HashMap, + res: &mut Vec, + seen: &mut HashSet, +) { + if seen.insert(c.id.original.to_string()) { + get_dependencies_from_type(&c.r#type, types, res, seen); + seen.remove(&c.id.original.to_string()); + } +} + fn get_dependencies( thing: &RustItem, types: &HashMap, @@ -130,6 +143,7 @@ fn get_dependencies( RustItem::Enum(en) => get_enum_dependencies(en, types, res, seen), RustItem::Struct(strct) => get_struct_dependencies(strct, types, res, seen), RustItem::Alias(alias) => get_type_alias_dependencies(alias, types, res, seen), + RustItem::Const(c) => get_const_dependencies(c, types, res, seen), } } @@ -194,6 +208,7 @@ pub(crate) fn topsort(things: &mut [RustItem]) { }, RustItem::Struct(strct) => strct.id.original.clone(), RustItem::Alias(ta) => ta.id.original.clone(), + RustItem::Const(c) => c.id.original.clone(), }; (id, thing) })); diff --git a/core/src/visitors.rs b/core/src/visitors.rs index e42b3a47..13c4273d 100644 --- a/core/src/visitors.rs +++ b/core/src/visitors.rs @@ -3,8 +3,8 @@ use crate::{ context::ParseContext, language::CrateName, parser::{ - has_typeshare_annotation, parse_enum, parse_struct, parse_type_alias, ErrorInfo, - ParseError, ParsedData, + has_typeshare_annotation, parse_const, parse_enum, parse_struct, parse_type_alias, + ErrorInfo, ParseError, ParsedData, }, rust_types::{RustEnumVariant, RustItem}, target_os_check::accept_target_os, @@ -136,6 +136,14 @@ impl<'a> TypeShareVisitor<'a> { .flat_map(|alias| alias.r#type.all_reference_type_names()), ); + // Constants + all_references.extend( + self.parsed_data + .consts + .iter() + .flat_map(|c| c.r#type.all_reference_type_names()), + ); + // Build a set of a all type names. let local_types = self .parsed_data @@ -297,6 +305,17 @@ impl<'ast> Visit<'ast> for TypeShareVisitor<'_> { syn::visit::visit_item_type(self, i); } + // Collect rust consts. + fn visit_item_const(&mut self, i: &'ast syn::ItemConst) { + debug!("Visiting {}", i.ident); + if has_typeshare_annotation(&i.attrs) && self.target_os_accepted(&i.attrs) { + debug!("\tParsing {}", i.ident); + self.collect_result(parse_const(i)); + } + + syn::visit::visit_item_const(self, i); + } + // Track potentially skipped modules. // fn visit_item_mod(&mut self, i: &'ast syn::ItemMod) { // if let Some(target_os) = self.target_os.as_ref() { diff --git a/core/tests/snapshot_tests.rs b/core/tests/snapshot_tests.rs index 8724de36..eb02798f 100644 --- a/core/tests/snapshot_tests.rs +++ b/core/tests/snapshot_tests.rs @@ -487,6 +487,7 @@ tests! { scala, typescript ]; + can_generate_const: [typescript, go, python]; can_generate_slice_of_user_type: [swift, kotlin, scala, typescript, go, python]; can_generate_readonly_fields: [ typescript