1 // this file implements the structures and lexer for the protocol buffer format 2 // required to parse a protocol buffer file or tree and generate 3 // code to read and write the specified format 4 module dprotobuf.pbroot; 5 import dprotobuf.pbgeneral; 6 import dprotobuf.pbmessage; 7 import dprotobuf.pbenum; 8 import dprotobuf.pbextension; 9 10 version(D_Version2) { 11 import std.algorithm; 12 import std.range; 13 } else 14 import dprotobuf.d1support; 15 16 import std..string; 17 import std.stdio; 18 19 struct PBRoot { 20 PBMessage[]message_defs; 21 PBEnum[]enum_defs; 22 string []imports; 23 string Package; 24 string[] comments; 25 PBExtension[]extensions; 26 27 static PBRoot opCall(string input) 28 in { 29 assert(!input.empty()); 30 } body { 31 PBRoot root; 32 auto pbstring = ParserData(input); 33 // rip off whitespace before looking for the next definition 34 pbstring = stripLWhite(pbstring); 35 CommentManager storeComment; 36 37 // loop until the string is gone 38 while(!pbstring.input.empty()) { 39 auto curElementType = typeNextElement(pbstring); 40 auto curElementLine = pbstring.line; 41 switch(curElementType){ 42 case PBTypes.PB_Package: 43 root.Package = parsePackage(pbstring); 44 break; 45 case PBTypes.PB_Message: 46 root.message_defs ~= PBMessage(pbstring); 47 break; 48 case PBTypes.PB_Extend: 49 root.extensions ~= PBExtension(pbstring); 50 break; 51 case PBTypes.PB_Enum: 52 root.enum_defs ~= PBEnum(pbstring); 53 break; 54 case PBTypes.PB_Comment: 55 // Preserve at least one spacing in comments 56 if(storeComment.line+1 < pbstring.line) 57 if(!storeComment.comments.empty()) 58 storeComment ~= ""; 59 storeComment ~= stripValidChars(CClass.Comment,pbstring); 60 storeComment.line = curElementLine; 61 if(curElementLine == storeComment.lastElementLine) 62 tryAttachComments(root, storeComment); 63 break; 64 case PBTypes.PB_MultiComment: 65 foreach(c; ripComment(pbstring)) 66 storeComment ~= c; 67 storeComment.line = pbstring.line; 68 break; 69 case PBTypes.PB_Option: 70 // rip of "option" and leading whitespace 71 pbstring.input.skipOver("option"); 72 pbstring = stripLWhite(pbstring); 73 ripOption(pbstring); 74 break; 75 case PBTypes.PB_Import: 76 pbstring = pbstring["import".length..pbstring.length]; 77 pbstring = stripLWhite(pbstring); 78 if (pbstring[0] != '"') throw new PBParseException("Root Definition("~root.Package~")","Imports must be quoted", pbstring.line); 79 // save imports for use by the compiler code 80 root.imports ~= ripQuotedValue(pbstring)[1..$-1]; 81 // ensure that the ; is removed 82 pbstring = stripLWhite(pbstring); 83 if (pbstring[0] != ';') throw new PBParseException("Root Definition("~root.Package~")","Missing ; after import \""~root.imports[$-1]~"\"", pbstring.line); 84 pbstring = pbstring[1..pbstring.length]; 85 pbstring = stripLWhite(pbstring); 86 break; 87 default: 88 throw new PBParseException("Root Definition("~root.Package~")","Either there's a definition here that isn't supported, or the definition isn't allowed here", pbstring.line); 89 } 90 pbstring.input.skipOver(";"); 91 // rip off whitespace before looking for the next definition 92 // this needs to stay at the end 93 pbstring = stripLWhite(pbstring); 94 storeComment.lastElementType = curElementType; 95 storeComment.lastElementLine = curElementLine; 96 tryAttachComments(root, storeComment); 97 98 } 99 return root; 100 } 101 102 static void tryAttachComments(ref PBRoot root, ref CommentManager storeComment) { 103 // Attach Comments to elements 104 if(!storeComment.comments.empty()) { 105 if(storeComment.line == storeComment.lastElementLine 106 || storeComment.line+3 > storeComment.lastElementLine) { 107 switch(storeComment.lastElementType) { 108 case PBTypes.PB_Comment: 109 case PBTypes.PB_MultiComment: 110 break; 111 case PBTypes.PB_Message: 112 root.message_defs[$-1].comments 113 = storeComment.comments; 114 goto default; 115 case PBTypes.PB_Enum: 116 root.enum_defs[$-1].comments 117 = storeComment.comments; 118 goto default; 119 case PBTypes.PB_Package: 120 root.comments 121 = storeComment.comments; 122 goto default; 123 default: 124 storeComment.comments = null; 125 } 126 } 127 } 128 } 129 130 static string parsePackage(ref ParserData pbstring) 131 in { 132 assert(pbstring.length); 133 } body { 134 pbstring = pbstring["package".length..pbstring.length]; 135 // strip any whitespace before the package name 136 pbstring = stripLWhite(pbstring); 137 // the next part of the string should be the package name up until the semicolon 138 string Package = stripValidChars(CClass.MultiIdentifier,pbstring); 139 // rip out any whitespace that might be here for some strange reason 140 pbstring = stripLWhite(pbstring); 141 // make sure the next character is a semicolon... 142 if (pbstring[0] != ';') { 143 throw new PBParseException("Package Definition","Whitespace is not allowed in package names.", pbstring.line); 144 } 145 // actually rip off the ; 146 pbstring = pbstring[1..pbstring.length]; 147 // make sure this is valid 148 if (!validateMultiIdentifier(Package)) throw new PBParseException("Package Identifier("~Package~")","Package identifier did not validate.", pbstring.line); 149 return Package; 150 } 151 152 } 153 154 // Verify comments attach to root/module/package 155 unittest { 156 string pbstr = "// Openning comments 157 package openning;"; 158 auto root = PBRoot(pbstr); 159 assert(root.Package == "openning"); 160 assert(root.comments.length == 1); 161 assert(root.comments[0] == "// Openning comments"); 162 } 163 164 // Verify multiline comments attach to root/module/package 165 unittest { 166 string pbstr = "/* Openning\ncomments */ 167 package openning;"; 168 auto root = PBRoot(pbstr); 169 assert(root.Package == "openning"); 170 assert(root.comments.length == 2); 171 assert(root.comments[0] == "/* Openning"); 172 assert(root.comments[1] == "comments */"); 173 } 174 175 unittest { 176 string pbstr = " 177 option optimize_for = SPEED; 178 package myfirstpackage; 179 // my comments hopefully won't explode anything 180 message Person {required string name= 1; 181 required int32 id =2; 182 optional string email = 3 ; 183 184 enum PhoneType{ 185 MOBILE= 0;HOME =1; 186 // gotta make sure comments work everywhere 187 WORK=2 ;} 188 189 message PhoneNumber { 190 required string number = 1; 191 //woah, comments in a sub-definition 192 optional PhoneType type = 2 ; 193 } 194 195 repeated PhoneNumber phone = 4; 196 } 197 //especially here 198 "; 199 200 writefln("unittest ProtocolBuffer.pbroot"); 201 auto root = PBRoot(pbstr); 202 assert(root.Package == "myfirstpackage"); 203 assert(root.message_defs[0].name == "Person"); 204 assert(root.message_defs[0].comments.length == 1); 205 assert(root.message_defs[0].message_defs[0].name == "PhoneNumber"); 206 assert(root.message_defs[0].enum_defs[0].name == "PhoneType"); 207 } 208 209 PBExtension[] getExtensions(T)(T root) { 210 PBExtension[]ret; 211 ret ~= root.extensions; 212 foreach(msg;root.message_defs) { 213 ret ~= getExtensions(msg); 214 } 215 return ret; 216 } unittest { 217 enum instr = 218 ParserData("extend Foo { 219 optional int blah = 1; 220 } 221 message Bar { 222 optional int de = 5; 223 extend FooBar { 224 optional string da = 2; 225 } 226 } 227 "); 228 229 assert(getExtensions(PBRoot(instr)).map!(x => x.name).equal(["Foo", "FooBar"])); 230 } 231 232 // this function digs through a given root to see if it has the message 233 // described by the dotstring 234 PBMessage* searchForMessage(ref PBRoot root, string message) { 235 return searchMessages(root, ParserData(message)); 236 } 237 238 /// Ditto 239 PBMessage* searchMessages(T)(ref T root, ParserData message) 240 in { 241 assert(message.length); 242 } body { 243 string name = stripValidChars(CClass.Identifier,message); 244 if (message.length) { 245 // rip off the leading . 246 message = message[1..message.length]; 247 } 248 // this is terminal, so run through the children to find a match 249 foreach(ref msg;root.message_defs) { 250 if (msg.name == name) { 251 if (!message.length) { 252 return &msg; 253 } else { 254 return searchMessages(msg,message); 255 } 256 } 257 } 258 return null; 259 } unittest { 260 enum instr = 261 ParserData(" 262 message Foo { 263 } 264 message Bar { 265 optional int de = 5; 266 } 267 "); 268 269 auto root = PBRoot(instr); 270 assert(searchForMessage(root, "Foo").name == "Foo"); 271 } 272 273 auto applyExtensions(PBRoot root, PBExtension[] exts) { 274 foreach(ext; exts) { 275 auto m = searchForMessage(root, ext.name); 276 if(m) { 277 *m = insertExtension(*m, ext); 278 break; 279 } 280 } 281 return root; 282 } static unittest { 283 enum foo = 284 ParserData("message Foo { 285 optional int de = 5; 286 extensions 1 to 4; 287 } 288 extend Foo { 289 optional int blah = 1; 290 } 291 "); 292 293 enum Foo = PBRoot(foo); 294 enum ExtFoo = Foo.getExtensions(); 295 296 applyExtensions(Foo, ExtFoo); 297 }