Accept 'acc' and 'drop' as token functions

'acc': accumulate tokens in the parser's state. This was previously
available by not specifying a token function.

'drop': just drop tokens when they're ready.

include/internal/yaml_parser.hrl

@@ -13,7 +13,7 @@
 -type yaml_parser_option() :: {default_tags, [{tag_uri(), tag_prefix()}]}
                             | {document_version, document_version()}
                             | {io_blocksize, pos_integer()}
-                            | {token_fun, yaml_parser_token_fun()}.
+                            | {token_fun, yaml_parser_token_fun() | acc | drop}.
 
 %% -------------------------------------------------------------------
 %% Main record to store the scanner state.
@@ -121,7 +121,11 @@
     %% parameters:
     %%   o  the block-context indentation prefixes (a stack);
     %%   o  the level of flow-context nesting (a stack too).
-    stream_state                      :: fun((#yaml_parser{}) ->
+    stream_state                      :: fun((string(),
+                                              pos_integer(),
+                                              pos_integer(),
+                                              non_neg_integer(),
+                                              #yaml_parser{}) ->
                                                {ok, #yaml_parser{}}
                                              | {continue, #yaml_parser{}}
                                              | none()),
@@ -160,7 +164,7 @@
     %%
 
     %% Callbacks.
-    token_fun :: yaml_parser_token_fun() | undefined,
+    token_fun = acc :: yaml_parser_token_fun() | acc | drop,
 
     %% List of scanned tokens with counters.
     tokens        = []  :: [yaml_token()],

src/yaml_constr.erl

@@ -47,7 +47,7 @@ last_chunk(Parser, More_Data) ->
 
 get_docs(Parser) ->
     case yaml_parser:get_token_fun(Parser) of
-        undefined ->
+        Not_Fun when Not_Fun == acc orelse Not_Fun == drop ->
             Error = #yaml_parsing_error{
               name = token_fun_cleared
             },

src/yaml_parser.erl

@@ -240,7 +240,7 @@ new(Source, Options) ->
       source       = Source,
       options      = Options,
       stream_state = fun start_stream/5,
-      token_fun    = proplists:get_value(token_fun, Options)
+      token_fun    = proplists:get_value(token_fun, Options, acc)
     }.
 
 next_chunk(Parser, <<>>, false) ->
@@ -3930,21 +3930,25 @@ handle_tag_property(Parser, Token) ->
     do_emit_token(Parser, Token).
 
 do_emit_token(
-  #yaml_parser{token_fun = undefined,
+  #yaml_parser{token_fun = Not_Fun,
     tks_queued = Queued, tks_first_idx = First,
     tks_emitted = Emitted, tks_ready = Ready} = Parser,
-  Token) ->
+  Token) when Not_Fun == acc orelse Not_Fun == drop ->
     %% The anchor was already counted when first removed from the queue.
     {Queued1, First1} = case ?TOKEN_NAME(Token) of
         yaml_anchor -> {Queued,     First};
         _           -> {Queued - 1, First + 1}
     end,
+    Ready1 = case Not_Fun of
+        acc  -> [Token | Ready];
+        drop -> Ready
+    end,
     Parser#yaml_parser{
       tks_queued    = Queued1,
       tks_first_idx = First1,
       tks_emitted   = Emitted + 1,
       last_token    = Token,
-      tks_ready     = [Token | Ready]
+      tks_ready     = Ready1
     };
 do_emit_token(
   #yaml_parser{token_fun = Fun,
@@ -4115,6 +4119,10 @@ is_option_valid({doc_version, {Major, Minor}}) when
 is_option_valid({io_blocksize, BS})
   when is_integer(BS) andalso BS >= 1 ->
     true;
+is_option_valid({token_fun, acc}) ->
+    true;
+is_option_valid({token_fun, drop}) ->
+    true;
 is_option_valid({token_fun, Fun})
   when is_function(Fun, 1) ->
     true;
@@ -4146,7 +4154,7 @@ invalid_option(Option) ->
             Error#yaml_invalid_option{
               text = "Invalid value for option \"token_fun\": "
               "it must be a function taking the parser state "
-              "as its sole argument"
+              "as its sole argument, or the atom 'acc' or 'drop'"
             };
         _ ->
             yaml_errors:format(Error, "Unknown option \"~w\"", [Option])