Merge pull request universal-ctags#4152 from masatake/nulltag

main: add nulltag/z, a new extra

Tmain/extras-long.d/stdout-expected.txt

@@ -7,6 +7,7 @@ p       pseudo              no      NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -25,6 +26,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -43,6 +45,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -61,6 +64,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -79,6 +83,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           yes     NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class

Tmain/json-output-format.d/stdout-expected.txt

@@ -20,6 +20,7 @@
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "fileScope", "pattern": "Include tags of file scope"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "guest", "pattern": "Include tags generated by guest parsers"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "inputFile", "pattern": "Include an entry for the base file name of every input file"}
+{"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "nulltag", "pattern": "Include tags with empty strings as their names"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "pseudo", "pattern": "Include pseudo tags"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "qualified", "pattern": "Include an extra class-qualified tag entry for each tag"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "reference", "pattern": "Include reference tags"}

Tmain/kind-abnormal-spec.d/stdout-expected.txt

@@ -12,6 +12,8 @@ r  emit a tag with multi roles
 R  emit a tag with multi roles(disabled by default) [off]
 f  tag for testing field:
 n  trigger notice output
+z  emit a tag having an empty string
+Z  don't emit a tag having an empty string
 
 # list kinds-full
 #LETTER NAME                                      ENABLED REFONLY NROLES MASTER DESCRIPTION
@@ -22,12 +24,14 @@ L       ThisShouldNotBePrintedKindNameMustBeGiven yes     no      0      NONE
 N       nothingSpecial                            yes     no      0      NONE   emit a normal tag
 Q       quit                                      yes     no      0      NONE   stop the parsing
 R       rolesDisabled                             no      yes     2      NONE   emit a tag with multi roles(disabled by default)
+Z       dontEmitNullTag                           yes     no      0      NONE   don't emit a tag having an empty string
 b       broken tag                                yes     no      1      NONE   name with unwanted characters
 d       disabled                                  no      no      2      NONE   a kind disabled by default
 e       enabled                                   yes     no      2      NONE   a kind enabled by default
 f       fieldMaker                                yes     no      0      NONE   tag for testing field:
 n       triggerNotice                             yes     no      0      NONE   trigger notice output
 r       roles                                     yes     yes     4      NONE   emit a tag with multi roles
+z       emitNullTag                               yes     no      0      NONE   emit a tag having an empty string
 
 # +K
 abnormal kindDefinition testing (no letter)	input.x	/^@$/;"	no letter

Tmain/list-extras.d/stdout-expected.txt

@@ -7,6 +7,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           yes     NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             yes     NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -25,6 +26,7 @@ p	pseudo	yes	NONE	no	Include pseudo tags
 q	qualified	yes	NONE	no	Include an extra class-qualified tag entry for each tag
 r	reference	yes	NONE	no	Include reference tags
 s	subparser	yes	NONE	no	Include tags generated by subparsers
+z	nulltag	yes	NONE	no	Include tags with empty strings as their names
 -	canonicalizedName	yes	Automake	no	Include canonicalized object name like libctags_a
 -	linkName	no	Fortran	no	Linking name used in foreign languages
 -	implicitClass	no	GDScript	no	Include tag for the implicitly defined unnamed class
@@ -43,3 +45,4 @@ p       pseudo    no      NONE     no    Include pseudo tags
 q       qualified no      NONE     no    Include an extra class-qualified tag entry for each tag
 r       reference no      NONE     no    Include reference tags
 s       subparser no      NONE     no    Include tags generated by subparsers
+z       nulltag   no      NONE     no    Include tags with empty strings as their names

Tmain/list-output-formats.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/list-output-formats.d/run.sh

@@ -0,0 +1,7 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+$CTAGS --quiet --options=NONE --machinable --with-list-header \
+       --list-output-formats | grep -v '^json'

Tmain/list-output-formats.d/stdout-expected.txt

@@ -0,0 +1,5 @@
+#OFORMAT	DEFAULT	AVAILABLE	NULLTAG
+e-ctags	no	yes	no
+etags	no	yes	no
+u-ctags	yes	yes	no
+xref	no	yes	yes

Tmain/nulltag-extra.d/input.cst

@@ -0,0 +1,3 @@
+Z
+z
+

Tmain/nulltag-extra.d/run.sh

@@ -0,0 +1,30 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+. ../utils.sh
+
+# The order of stdout and stderr lines is not stable.
+exit_if_win32 "$CTAGS"
+
+# is_feature_available $CTAGS json
+
+O="--options=NONE --language-force=CTagsSelfTest"
+
+for fmt in xref; do
+	echo "# no extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" input.cst 2>&1
+
+	echo "# drop '0' extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=-z input.cst 2>&1
+
+	echo "# drop '{nulltag}' extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=-'{nulltag}' input.cst 2>&1
+
+	echo '# with --extras=+0 ($fmt)'
+	${CTAGS} $O -o - --output-format="$fmt" --extras=+z input.cst 2>&1
+
+	echo "# with --extras=+{nulltag}' ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=+'{nulltag}' input.cst 2>&1
+done | sed -e 's/\.exe//'

Tmain/nulltag-extra.d/stdout-expected.txt

@@ -0,0 +1,17 @@
+# no extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# drop '0' extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# drop '{nulltag}' extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# with --extras=+0 ($fmt)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+                 emitNullTag    2 input.cst        z
+# with --extras=+{nulltag}' (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+                 emitNullTag    2 input.cst        z

Units/parser-javascript.r/js-extract-empty-property-in-json.d/args.ctags

@@ -0,0 +1,4 @@
+--sort=no
+--extras=+{nulltag}
+--fields=+{extras}
+--output-format=json

Units/parser-javascript.r/js-extract-empty-property-in-json.d/expected.tags-json

@@ -0,0 +1,2 @@
+{"_type": "tag", "name": "", "path": "input.js", "pattern": "/^var variable = { \"\": \"value\" };$/", "kind": "property", "scope": "variable", "scopeKind": "variable", "extras": "nulltag"}
+{"_type": "tag", "name": "variable", "path": "input.js", "pattern": "/^var variable = { \"\": \"value\" };$/", "kind": "variable"}

Units/parser-javascript.r/js-extract-empty-property-in-json.d/features

@@ -0,0 +1 @@
+json

Units/parser-javascript.r/js-extract-empty-property-in-json.d/input.js

@@ -0,0 +1 @@
+var variable = { "": "value" };

Units/parser-javascript.r/js-extract-empty-property.d/args.ctags

@@ -0,0 +1,5 @@
+--sort=no
+--extras=+{nulltag}
+--fields=+{extras}
+-x
+--_xformat=%-16N %-10K %4n input.js %C

Units/parser-javascript.r/js-extract-empty-property.d/expected.tags-x

@@ -0,0 +1,2 @@
+                 property      1 input.js var variable = { "": "value" };
+variable         variable      1 input.js var variable = { "": "value" };

Units/parser-javascript.r/js-extract-empty-property.d/input.js

@@ -0,0 +1 @@
+var variable = { "": "value" };

Units/parser-javascript.r/js-skip-empty-property.d/args.ctags

@@ -0,0 +1 @@
+--sort=no

Units/parser-javascript.r/js-skip-empty-property.d/expected.tags

@@ -0,0 +1 @@
+variable	input.js	/^var variable = { "": "value" };$/;"	v

Units/parser-javascript.r/js-skip-empty-property.d/input.js

@@ -0,0 +1 @@
+var variable = { "": "value" };

docs/internal.rst

@@ -964,6 +964,26 @@ Refer `peg/valink.peg
 <https://github.com/universal-ctags/ctags/blob/master/peg/varlink.peg>`_ as a
 sample of a parser using PackCC.
 
+Null tags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+*Null tag* is a tag having an empty string as its name.
+
+Universal Ctags supports null tags cautiously.
+See the description for ``nulltag``/``z`` extra in :ref:`ctags(1) <ctags(1)>`.
+
+If you want to make a null tag, set 1 to ``allowNullTag`` of
+``tagEtnryInfo`` before calling ``makeTagEntry``.  ``makeTagEntry``
+evaluates the member. The following pseudo code doesn't work:
+
+.. code-block:: C
+
+   tagEntryInfo e;
+   /* ... */
+   int corkIndex = makeTagEntry (&e);
+   /* ... */
+   tagEntryInfo *p = getEntryInCorkQueue (corkIndex);
+   p->allowNullTag = 1;
+
 Automatic parser guessing (TBW)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/man/ctags.1.rst

@@ -318,6 +318,8 @@ Output Format Options
 	the ctags executable is built with ``libjansson``.
 	See :ref:`ctags-json-output(5) <ctags-json-output(5)>` for more about ``json`` format.
 
+	See also ``--list-output-formats``.
+
 ``-e``
 	Same as ``--output-format=etags``.
 	Enable etags mode, which will create a tag file for use with the Emacs
@@ -1236,6 +1238,14 @@ Listing Options
 	definition.
 	See :ref:`ctags-optlib(7) <ctags-optlib(7)>`.
 
+``--list-output-formats``
+	Lists the output formats that can be used in ``--output-format`` option.
+
+	``NULLTAG`` column represetns whether the format supports *null tags* or
+	not. See ``nulltag``/``z`` in "`Extras`_" about the null tags.
+
+	(since verison 6.2.0)
+
 ``--list-params[=(<language>|all)]``
 	Lists the parameters for either the specified *<language>* or ``all``
 	languages, and then exits.
@@ -1835,6 +1845,8 @@ roles of tags to include in the output file for a particular language.
 Inquire the output of "``ctags --list-roles``" for the list of
 roles.
 
+.. _extras:
+
 Extras
 ~~~~~~
 
@@ -1927,6 +1939,25 @@ The meaning of major extras is as follows (long-name flag/one-letter flag):
 
 	The etags mode enables the ``Unknown`` parser implicitly.
 
+``nulltag``/``z``
+	Include tags (*null tags*) having empty strings as their names.
+	Generally speaking, trying to make a null tag is a sign of a parser bug
+	or broken input. ctags warns such trying or throws the
+	null tag away. To suppress the warnings, use ``--quiet`` option.
+
+	On the other hand, null tags are valid in some languages.
+	Consider ``{"":  val}`` in a JavaScript sourece code. The empty string is
+	valid as a key. If a parser intentionally makes a null tag (a valid null tag),
+	ctags doesn't warn but discard it by default.
+
+	The discards are due because some output formats may not consider null tags.
+
+	With ``nulltag``/``z`` extra, you can force ctags to emit the nulltags. This extra
+	is effective only if the output format supports null tags. ``--list-output-formats``
+	option tells you which output formats support null tags.
+
+	(since version 6.2.0)
+
 ``pseudo``/``p``
 	Include pseudo-tags. Enabled by default unless the tag file is
 	written to standard output. See :ref:`ctags-client-tools(7) <ctags-client-tools(7)>` about
@@ -2273,6 +2304,8 @@ The official Universal Ctags web site at: https://ctags.io/
 Also ``ex(1)``, ``vi(1)``, ``elvis(1)``, or, better yet, ``vim(1)``, the official editor of ctags.
 For more information on ``vim(1)``, see the Vim web site at: https://www.vim.org/
 
+About the file format for ``TAGS``, see `emacs git
+<https://git.savannah.gnu.org/cgit/emacs.git/tree/etc/ETAGS.EBNF>`_.
 
 AUTHOR
 ------

docs/news/HEAD.rst

@@ -5,6 +5,20 @@ Changes in 6.?.0
 New and extended options and their flags
 ---------------------------------------------------------------------
 
+``--list-output-formats`` option
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+See :ref:`option_listing` in :ref:`ctags(1) <ctags(1)>`.
+
+``nulltag``/``z`` extra
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Universal Ctags now supports tags (*null tags*) having empty strings as their names.
+See :ref:`extras` in :ref:`ctags(1) <ctags(1)>`.
+
+.. note::
+
+   * ``libreadtags`` and ``readtags`` do not support the null tags yet.
+   * ``json`` and ``xref`` output formats support the null tags.
+
 Incompatible changes
 ---------------------------------------------------------------------
 

docs/output-format.rst

@@ -23,11 +23,8 @@ Supported *format* are ``u-ctags``, ``e-ctags``, ``etags``, ``xref``, and ``json
 
 ``etags``
 	Output format for Emacs etags.
-	``--output-format=etags`` can be abbreviated as ``-e``.
 
-	See `emacs git
-	<https://git.savannah.gnu.org/cgit/emacs.git/tree/etc/ETAGS.EBNF>`_ for
-	details.
+	See the description of ``-e`` option in :ref:`ctags(1) <ctags(1)>`.
 
 ``xref``
 	A tabular, human-readable cross reference (xref) format.
@@ -38,7 +35,7 @@ Supported *format* are ``u-ctags``, ``e-ctags``, ``etags``, ``xref``, and ``json
 ``json``
 	JSON format.
 
-	See section :ref:`output-json` for details.
+	See :ref:`ctags-client-tools(7) <ctags-client-tools(7)>`.
 
 *********
 
@@ -47,4 +44,3 @@ Supported *format* are ``u-ctags``, ``e-ctags``, ``etags``, ``xref``, and ``json
 
 	output-tags.rst
 	output-xref.rst
-	output-json.rst

main/colprint.c

@@ -19,9 +19,10 @@
 
 
 enum colprintJustification {
-	COLPRINT_LEFT,				/* L:... */
-	COLPRINT_RIGHT,				/* R:... */
-	COLPRINT_LAST,
+	COLPRINT_LEFT  = 0,
+	COLPRINT_RIGHT = 1,
+	COLPRINT_JUST  = 1 << 0,
+	COLPRINT_LAST  = 1 << 1,	/* private use */
 };
 
 struct colprintHeaderColumn {
@@ -101,7 +102,7 @@ struct colprintTable *colprintTableNew (const char* columnHeader, ... /* NULL TE
 
 	struct colprintHeaderColumn *last_col =	ptrArrayLast (table->header);
 	if (last_col)
-		last_col->justification = COLPRINT_LAST;
+		last_col->justification |= COLPRINT_LAST;
 
 	return table;
 }
@@ -130,7 +131,7 @@ static void colprintColumnPrintGeneric (vString *column, struct colprintHeaderCo
 	if (machinable)
 	{
 		fputs (vStringValue (column), fp);
-		if (spec->justification != COLPRINT_LAST)
+		if (! (spec->justification & COLPRINT_LAST))
 			fputc ('\t', fp);
 	}
 	else
@@ -139,17 +140,16 @@ static void colprintColumnPrintGeneric (vString *column, struct colprintHeaderCo
 		size_t colLen = vStringLength (column);
 		if (colLen < maxWidth)
 			padLen = maxWidth - colLen;
-		if (spec->justification == COLPRINT_LEFT
-			|| spec->justification == COLPRINT_LAST)
+		if ((spec->justification & COLPRINT_JUST) == COLPRINT_LEFT)
 		{
 			fputs (vStringValue (column), fp);
-			if (spec->justification != COLPRINT_LAST)
+			if (! (spec->justification & COLPRINT_LAST))
 			{
 				fillWithWhitespaces (padLen, fp);
 				fputc (' ', fp);
 			}
 		}
-		else
+		else if ((spec->justification & COLPRINT_JUST) == COLPRINT_RIGHT)
 		{
 			fillWithWhitespaces (padLen, fp);
 			fputs (vStringValue (column), fp);