Add support for Thrift annotations (#276)

Annotations appear within parentheses as part of various Thrift IDL
types such as fields and structures. They can be used to hint the
compiler or inform runtime behavior; the specification is pretty
open-ended.

This change adds support for parsing annotations (whose presence
previously resulted in parse errors) and storing them on our parser
models.

We currently support annotations on:

 - Enums
 - Fields
 - Functions
 - Services
 - Structures, Unions, and Exceptions

We can parse annotations on typedefs but don't currently store them
anywhere due to the way we internally represent a schema's typedefs as a
simple map of strings to types. We can revisit that later, perhaps as
part of a broader typedef refactoring.

lib/thrift/parser.ex

@@ -9,6 +9,9 @@ defmodule Thrift.Parser do
   @typedoc "A Thrift IDL line number"
   @type line :: pos_integer | nil
 
+  @typedoc "A map of Thrift annotation keys to values"
+  @type annotations :: %{String.t => String.t}
+
   @typedoc "A schema path element"
   @type path_element :: String.t | atom
 

lib/thrift/parser/models.ex

@@ -8,8 +8,6 @@ defmodule Thrift.Parser.Models do
   import Thrift.Parser.Conversions
   alias Thrift.Parser.{Literals, Types}
 
-  @type line :: nil | pos_integer
-
   defmodule Namespace do
     @moduledoc """
     A Thrift namespace.
@@ -70,10 +68,14 @@ defmodule Thrift.Parser.Models do
     """
 
     @type enum_value :: bitstring | integer
-    @type t :: %TEnum{line: Parser.line, name: atom, values: [{atom, enum_value}]}
+    @type t :: %TEnum{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      name: atom,
+      values: [{atom, enum_value}]}
 
     @enforce_keys [:name, :values]
-    defstruct line: nil, name: nil, values: []
+    defstruct line: nil, annotations: %{}, name: nil, values: []
 
     @spec new(charlist, %{charlist => enum_value}) :: t
     def new(name, values) do
@@ -105,11 +107,18 @@ defmodule Thrift.Parser.Models do
     """
 
     @type printable :: String.t | atom
-    @type t :: %Field{line: Parser.line, id: integer, name: atom, type: Types.t,
-                      required: boolean, default: Literals.t}
+    @type t :: %Field{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      id: integer,
+      name: atom,
+      type: Types.t,
+      required: boolean,
+      default: Literals.t}
 
     @enforce_keys [:id, :name, :type]
-    defstruct line: nil, id: nil, name: nil, type: nil, required: :default, default: nil
+    defstruct line: nil, annotations: %{}, id: nil, name: nil, type: nil,
+      required: :default, default: nil
 
     @spec new(integer, boolean, Types.t, charlist, Literals.t) :: t
     def new(id, required, type, name, default) do
@@ -166,10 +175,14 @@ defmodule Thrift.Parser.Models do
     Exceptions can happen when the remote service encounters an error.
     """
 
-    @type t :: %Exception{line: Parser.line, name: atom, fields: [Field.t]}
+    @type t :: %Exception{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      name: atom,
+      fields: [Field.t]}
 
     @enforce_keys [:name, :fields]
-    defstruct line: nil, fields: %{}, name: nil
+    defstruct line: nil, annotations: %{}, fields: %{}, name: nil
 
     @spec new(charlist, [Field.t, ...]) :: t
     def new(name, fields) do
@@ -187,10 +200,14 @@ defmodule Thrift.Parser.Models do
     The basic datastructure in Thrift, structs have aa name and a field list.
     """
 
-    @type t :: %Struct{line: Parser.line, name: atom, fields: [Field.t]}
+    @type t :: %Struct{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      name: atom,
+      fields: [Field.t]}
 
     @enforce_keys [:name, :fields]
-    defstruct line: nil, name: nil, fields: %{}
+    defstruct line: nil, annotations: %{}, name: nil, fields: %{}
 
     @spec new(charlist, [Field.t, ...]) :: t
     def new(name, fields) do
@@ -208,10 +225,14 @@ defmodule Thrift.Parser.Models do
     Unions can have one field set at a time.
     """
 
-    @type t :: %Union{line: Parser.line, name: atom, fields: [Field.t]}
+    @type t :: %Union{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      name: atom,
+      fields: [Field.t]}
 
     @enforce_keys [:name, :fields]
-    defstruct line: nil, name: nil, fields: %{}
+    defstruct line: nil, annotations: %{}, name: nil, fields: %{}
 
     @spec new(charlist, [Field.t, ...]) :: t
     def new(name, fields) do
@@ -304,11 +325,18 @@ defmodule Thrift.Parser.Models do
     """
 
     @type return :: :void | Types.t
-    @type t :: %Function{line: Parser.line, oneway: boolean, return_type: return, name: atom,
-                         params: [Field.t], exceptions: [Exception.t]}
+    @type t :: %Function{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      oneway: boolean,
+      return_type: return,
+      name: atom,
+      params: [Field.t],
+      exceptions: [Exception.t]}
 
     @enforce_keys [:name]
-    defstruct line: nil, oneway: false, return_type: :void, name: nil, params: [], exceptions: []
+    defstruct line: nil, annotations: %{}, oneway: false, return_type: :void,
+      name: nil, params: [], exceptions: []
 
     @spec new(boolean, Types.t, charlist, [Field.t, ...], [Exception.t, ...]) :: t
     def new(oneway, return_type, name, params, exceptions) do
@@ -332,10 +360,16 @@ defmodule Thrift.Parser.Models do
     Services hold RPC functions and can extend other services.
     """
 
-    @type t :: %Service{line: Parser.line, name: atom, extends: atom, functions: %{atom => Function.t}}
+    @type t :: %Service{
+      line: Parser.line,
+      annotations: Parser.annotations,
+      name: atom,
+      extends: atom,
+      functions: %{atom => Function.t}}
 
     @enforce_keys [:name, :functions]
-    defstruct line: nil, name: nil, extends: nil, functions: %{}
+    defstruct line: nil, annotations: %{}, name: nil, extends: nil,
+      functions: %{}
 
     @spec new(charlist, [Function.t, ...], charlist) :: t
     def new(name, functions, extends) do

src/thrift_parser.yrl

@@ -25,6 +25,7 @@ Nonterminals
     FunctionList Function Oneway ReturnType Throws
     FieldList Field FieldIdentifier FieldRequired FieldDefault
     FieldType BaseType MapType SetType ListType
+    Annotations Annotation AnnotationList
     Separator.
 
 Terminals
@@ -99,39 +100,39 @@ ConstList -> ConstValue Separator ConstList: ['$1'|'$3'].
 
 % Typedef
 
-Typedef -> typedef FieldType ident Separator:
-    {typedef, '$2', unwrap('$3')}.
+Typedef -> typedef FieldType Annotations ident Annotations Separator:
+    {typedef, '$2', unwrap('$4')}.
 
 % Enum
 
-Enum -> enum ident '{' EnumList '}':
-    build_model('TEnum', line('$1'), [unwrap('$2'), '$4']).
+Enum -> enum ident '{' EnumList '}' Annotations:
+    build_model('TEnum', line('$1'), '$6', [unwrap('$2'), '$4']).
 
 EnumList -> EnumValue Separator: ['$1'].
 EnumList -> EnumValue Separator EnumList: ['$1'|'$3'].
 
-EnumValue -> ident '=' int: {unwrap('$1'), unwrap('$3')}.
-EnumValue -> ident: unwrap('$1').
+EnumValue -> ident '=' int Annotations: {unwrap('$1'), unwrap('$3')}.
+EnumValue -> ident Annotations: unwrap('$1').
 
 % Struct
 
-Struct -> struct ident '{' FieldList '}':
-    build_model('Struct', line('$1'), [unwrap('$2'), '$4']).
+Struct -> struct ident '{' FieldList '}' Annotations:
+    build_model('Struct', line('$1'), '$6', [unwrap('$2'), '$4']).
 
 % Union
 
-Union -> union ident '{' FieldList '}':
-    build_model('Union', line('$1'), [unwrap('$2'), '$4']).
+Union -> union ident '{' FieldList '}' Annotations:
+    build_model('Union', line('$1'), '$6', [unwrap('$2'), '$4']).
 
 % Exception
 
-Exception -> exception ident '{' FieldList '}':
-    build_model('Exception', line('$1'), [unwrap('$2'), '$4']).
+Exception -> exception ident '{' FieldList '}' Annotations:
+    build_model('Exception', line('$1'), '$6', [unwrap('$2'), '$4']).
 
 % Service
 
-Service -> service ident Extends '{' FunctionList '}':
-    build_model('Service', line('$1'), [unwrap('$2'), '$5', '$3']).
+Service -> service ident Extends '{' FunctionList '}' Annotations:
+    build_model('Service', line('$1'), '$7', [unwrap('$2'), '$5', '$3']).
 
 Extends -> extends ident: unwrap('$2').
 Extends -> '$empty': nil.
@@ -141,8 +142,8 @@ Extends -> '$empty': nil.
 FunctionList -> '$empty': [].
 FunctionList -> Function FunctionList: ['$1'|'$2'].
 
-Function -> Oneway ReturnType ident '(' FieldList ')' Throws Separator:
-    build_model('Function', line('$3'), ['$1', '$2', unwrap('$3'), '$5', '$7']).
+Function -> Oneway ReturnType ident '(' FieldList ')' Throws Annotations Separator:
+    build_model('Function', line('$3'), '$8', ['$1', '$2', unwrap('$3'), '$5', '$7']).
 
 Oneway -> '$empty': false.
 Oneway -> oneway: true.
@@ -158,8 +159,8 @@ Throws -> throws '(' FieldList ')': '$3'.
 FieldList -> '$empty': [].
 FieldList -> Field FieldList: ['$1'|'$2'].
 
-Field -> FieldIdentifier FieldRequired FieldType ident FieldDefault Separator:
-    build_model('Field', line('$4'), ['$1', '$2', '$3', unwrap('$4'), '$5']).
+Field -> FieldIdentifier FieldRequired FieldType ident FieldDefault Annotations Separator:
+    build_model('Field', line('$4'), '$6', ['$1', '$2', '$3', unwrap('$4'), '$5']).
 
 FieldIdentifier -> int ':': unwrap('$1').
 FieldIdentifier -> '$empty': nil.
@@ -191,7 +192,20 @@ BaseType -> binary: binary.
 
 MapType -> map '<' FieldType ',' FieldType '>': {'$3', '$5'}.
 SetType -> set '<' FieldType '>': '$3'.
-ListType -> list '<' FieldType '>': '$3'.
+ListType -> list '<' FieldType Annotations '>': '$3'.
+
+% Annotations
+
+Annotations -> '(' AnnotationList ')': '$2'.
+Annotations -> '$empty': #{}.
+
+Annotation -> ident Separator:
+    #{list_to_atom(unwrap('$1')) => <<"1">>}.
+Annotation -> ident '=' string Separator:
+    #{list_to_atom(unwrap('$1')) => list_to_binary(unwrap('$3'))}.
+
+AnnotationList -> '$empty': #{}.
+AnnotationList -> Annotation AnnotationList: maps:merge('$1', '$2').
 
 % Separator
 
@@ -210,6 +224,10 @@ build_model(Type, Args) when is_list(Args) ->
 build_model(Type, Line, Args) when is_integer(Line) and is_list(Args) ->
     Model = build_model(Type, Args),
     maps:put(line, Line, Model).
+build_model(Type, Line, Annotations, Args)
+  when is_integer(Line) and is_map(Annotations) and is_list(Args) ->
+    Model = build_model(Type, Line, Args),
+    maps:put(annotations, Annotations, Model).
 
 % Extract the line number from the lexer's expression tuple.
 line({_Token, Line}) -> Line;

test/fixtures/app/thrift/AnnotationTest.thrift

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+typedef list<i32> ( cpp.template = "std::list" ) int_linked_list
+
+struct foo {
+  1: i32 bar ( presence = "required" );
+  2: i32 baz ( presence = "manual", cpp.use_pointer = "", );
+  3: i32 qux;
+  4: i32 bop;
+} (
+  cpp.type = "DenseFoo",
+  python.type = "DenseFoo",
+  java.final = "",
+  annotation.without.value,
+)
+
+exception foo_error {
+  1: i32 error_code ( foo="bar" )
+  2: string error_msg
+} (foo = "bar")
+
+typedef string ( unicode.encoding = "UTF-16" ) non_latin_string (foo="bar")
+typedef list< double ( cpp.fixed_point = "16" ) > tiny_float_list
+
+enum weekdays {
+  SUNDAY ( weekend = "yes" ),
+  MONDAY,
+  TUESDAY,
+  WEDNESDAY,
+  THURSDAY,
+  FRIDAY,
+  SATURDAY ( weekend = "yes" )
+} (foo.bar="baz")
+
+service foo_service {
+  void foo() ( foo = "bar" )
+} (a.b="c")
+

test/mix/tasks/compile.thrift_test.exs

@@ -14,7 +14,8 @@ defmodule Mix.Tasks.Compile.ThriftTest do
     in_fixture fn ->
       with_project_config [], fn ->
         assert run([]) =~ """
-          Compiling 3 files (.thrift)
+          Compiling 4 files (.thrift)
+          Compiled thrift/AnnotationTest.thrift
           Compiled thrift/StressTest.thrift
           Compiled thrift/ThriftTest.thrift
           Compiled thrift/numbers.thrift

test/thrift/parser/annotation_test.exs

@@ -0,0 +1,53 @@
+defmodule Thrift.Parser.ParserTest do
+  use ExUnit.Case, async: true
+  import Thrift.Parser, only: [parse: 1]
+
+  setup_all do
+    {:ok, schema} =
+      "test/fixtures/app/thrift/AnnotationTest.thrift"
+      |> File.read!
+      |> parse
+    {:ok, [schema: schema]}
+  end
+
+  defp find_field(fields, name) do
+    Enum.find(fields, &match?(%{name: ^name}, &1))
+  end
+
+  test "enum annotations", context do
+    assert %{enums: %{weekdays: enum}} = context[:schema]
+    assert enum.annotations == %{:"foo.bar" => "baz"}
+  end
+
+  test "struct annotations", context do
+    assert %{structs: %{foo: struct}} = context[:schema]
+    assert struct.annotations == %{
+      :"cpp.type" => "DenseFoo",
+      :"python.type" => "DenseFoo",
+      :"java.final" => "",
+      :"annotation.without.value" => "1"}
+
+    assert bar = find_field(struct.fields, :bar)
+    assert bar.annotations == %{:presence => "required"}
+    assert baz = find_field(struct.fields, :baz)
+    assert baz.annotations == %{:presence => "manual", :"cpp.use_pointer" => ""}
+  end
+
+  test "service annotations", context do
+    assert %{services: %{foo_service: service}} = context[:schema]
+    assert service.annotations == %{:"a.b" => "c"}
+  end
+
+  test "function annotations", context do
+    assert %{services: %{foo_service: service}} = context[:schema]
+    assert %{functions: %{foo: function}} = service
+    assert function.annotations == %{:foo => "bar"}
+  end
+
+  test "exception annotations", context do
+    assert %{exceptions: %{foo_error: exception}} = context[:schema]
+    assert exception.annotations == %{:foo => "bar"}
+    assert field = find_field(exception.fields, :error_code)
+    assert field.annotations == %{:foo => "bar"}
+  end
+end